Voyager 1 retired: a push to master no longer deploys

Its database was deleted on 2026-10-09; a deploy would start it on an
empty one. The deploy job is removed (tests still run), and homelab's
~/bin/deploy-usenet-indexer refuses voyager 1 deploys (voyager 2 deploys
from branch rebuild with deploy-rebuild).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitea/workflows/deploy.yaml

@@ -1,9 +1,8 @@
-# Tests every push; a push to master that passes is deployed to homelab.
-#
-# Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
-# run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
-# which fetches the commit from this repo, builds the image, runs the
-# migrations and recreates the app services (never postgres or redis).
+# Tests every push. Voyager 1 (this branch) is retired since 2026-10-11: its
+# database was deleted on 2026-10-09, and a push to master no longer deploys.
+# Voyager 2 (branch rebuild) has its own workflow and deploy
+# (deploy/README.md there); homelab's ~/bin/deploy-usenet-indexer refuses
+# voyager 1 deploys too.
 name: Test and deploy
 
 on:
@@ -63,19 +62,3 @@ jobs:
       - name: Test
         run: node ace test
 
-  deploy:
-    needs: test
-    if: github.ref == 'refs/heads/master'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Deploy to homelab
-        env:
-          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
-        run: |
-          install -m 700 -d ~/.ssh
-          printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
-          ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
-            daan@10.206.0.175 "deploy ${{ github.sha }}"

app/controllers/seed_controller.ts

@@ -1,6 +1,7 @@
 import type { HttpContext } from '@adonisjs/core/http'
 import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator'
 import SeedFetchPool from '#services/SeedFetchPool'
+import RemoteContentService, { ArticleRelayOff, JobLost } from '#services/RemoteContentService'
 import type { SeedBatch } from '#lib/seed/wire'
 
 /**
@@ -71,11 +72,78 @@ export default class SeedController {
     })
   }
 
+  /**
+   * POST /api/v1/seed/inspect/lease { max } → 200 { jobs } or 204: content
+   * inspection and archive walks for the node (CONCEPT.md D41).
+   */
+  async inspectLease({ seedNode, request, response }: HttpContext) {
+    const max = Math.min(10, Math.max(1, Number(request.input('max')) || 1))
+    const jobs = await RemoteContentService.lease(seedNode.id, max)
+    return jobs.length > 0 ? { jobs } : response.noContent()
+  }
+
+  /**
+   * POST /api/v1/seed/inspect/done { queue, jobId, token, result } where
+   * result is the probe as a JSON string (the body parser would turn the
+   * empty strings inside it into nulls).
+   */
+  async inspectDone({ request, response }: HttpContext) {
+    const { queue, jobId, token, result } = request.only(['queue', 'jobId', 'token', 'result'])
+    if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string' || typeof result !== 'string') {
+      return response.badRequest({ error: 'Expected { queue, jobId, token, result: JSON string }' })
+    }
+    return this.job(response, () => RemoteContentService.finish(queue, jobId, token, JSON.parse(result)))
+  }
+
+  /** POST /api/v1/seed/inspect/failed { queue, jobId, token, error } */
+  async inspectFailed({ request, response }: HttpContext) {
+    const { queue, jobId, token, error } = request.only(['queue', 'jobId', 'token', 'error'])
+    if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
+      return response.badRequest({ error: 'Expected { queue, jobId, token, error }' })
+    }
+    return this.job(response, () => RemoteContentService.fail(queue, jobId, token, String(error ?? 'unknown error')))
+  }
+
+  /** POST /api/v1/seed/inspect/release { queue, jobId, token }: a stopping node gives a job back. */
+  async inspectRelease({ request, response }: HttpContext) {
+    const { queue, jobId, token } = request.only(['queue', 'jobId', 'token'])
+    if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
+      return response.badRequest({ error: 'Expected { queue, jobId, token }' })
+    }
+    return this.job(response, () => RemoteContentService.release(queue, jobId, token))
+  }
+
+  /** GET /api/v1/seed/articles?id=<message id> → the raw article body; 410 when the server doesn't have it. */
+  async article({ request, response }: HttpContext) {
+    const id = request.input('id')
+    if (typeof id !== 'string' || id.length === 0 || id.length > 250) return response.badRequest({ error: 'Expected ?id=<message id>' })
+    try {
+      const body = await RemoteContentService.article(id)
+      if (!body) return response.gone({ error: 'No such article' })
+      response.header('content-type', 'application/octet-stream')
+      return response.send(body)
+    } catch (error: any) {
+      if (error instanceof ArticleRelayOff) return response.serviceUnavailable({ error: error.message })
+      return response.badGateway({ error: `NNTP: ${error.message}` })
+    }
+  }
+
   /** GET /api/v1/seed/status */
   async status() {
     return SeedOrchestrator.status()
   }
 
+  /** 409 when the job's lock is gone (it was given to another node meanwhile). */
+  private async job(response: HttpContext['response'], action: () => Promise<unknown>) {
+    try {
+      await action()
+      return { ok: true }
+    } catch (error) {
+      if (error instanceof JobLost) return response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
   /** 409 when the range was given to another node meanwhile; the node should drop it. */
   private async leased(response: HttpContext['response'], action: () => Promise<unknown>) {
     try {

app/services/ArchiveService.ts

@@ -1,12 +1,8 @@
-import { execFile } from 'node:child_process'
-import { mkdtemp, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import path from 'node:path'
-import { promisify } from 'node:util'
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
+import FileStore from '#services/FileStore'
 import IngestService from '#services/IngestService'
-import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
+import type { RarInfo } from '#lib/rar/index'
 import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
 
 /** Stored per file and release; a listing of thousands of files is not useful. */
@@ -27,8 +23,9 @@ export interface ArchiveResult {
  */
 class ArchiveService {
   async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
-    const stored = { ...info, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
-    await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) })
+    const { next: _next, ...listing } = info
+    const stored = { ...listing, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
+    await FileStore.setDetails(Number(file.id), { archive: stored })
 
     let releaseId = file.release_id ?? binary?.release_id ?? null
     let grouped = false
@@ -70,43 +67,6 @@ class ArchiveService {
   }
 }
 
-const run = promisify(execFile)
-
-/** Tools that print one archive entry to stdout, tried in order. */
-const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
-  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
-  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
-]
-
-/**
- * The unpacked data of one entry of a complete (single-volume) archive:
- * copied out when stored, else unpacked with bsdtar (libarchive, in the
- * Docker image) or unrar. Null when that is not possible.
- */
-export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
-  const stored = storedEntryData(archive, entry)
-  if (stored) return stored
-  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
-  if (entry.size !== null && entry.size > maxBytes) return null
-
-  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
-  try {
-    const file = path.join(dir, 'archive.rar')
-    await writeFile(file, archive)
-    for (const [tool, args] of EXTRACTORS) {
-      try {
-        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
-        if (stdout.length > 0) return stdout
-      } catch (error: any) {
-        if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
-      }
-    }
-    return null
-  } finally {
-    await rm(dir, { recursive: true, force: true })
-  }
-}
-
 /**
  * A release name from an archive listing: the top-level directory shared by
  * all entries when it reads like a release name, otherwise the stem of the
@@ -114,9 +74,11 @@ export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes =
  * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
  */
 export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
-  const entries = info.entries.filter((e) => !e.directory)
-  const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0]))
-  if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) {
+  // Hidden entries name nothing: one poster adds the same 22 GB ".1917-….mkv" decoy to every upload (LOG.md 2026-10-06 (2)).
+  const listed = info.entries.filter((e) => !isHidden(e.name))
+  const entries = listed.filter((e) => !e.directory)
+  const tops = new Set(listed.map((e) => e.name.split(/[\\/]/)[0]))
+  if (tops.size === 1 && listed.some((e) => /[\\/]/.test(e.name))) {
     const [top] = tops
     if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
   }
@@ -128,6 +90,36 @@ export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string
   return candidates[0]?.stem
 }
 
+/** An entry in a hidden directory or a hidden file itself (`.x/…`, `dir/.x`). */
+export function isHidden(name: string): boolean {
+  return name.split(/[\\/]/).some((part) => part.startsWith('.'))
+}
+
+/**
+ * Releases named after a hidden archive entry before those were skipped:
+ * renamed from their stored listing (no downloads). Used by
+ * `release:rename --hidden`.
+ */
+export async function renameHiddenNamed(): Promise<number> {
+  const releases = await db
+    .from('releases')
+    .where('name', 'like', '.%')
+    .whereNot('source', 'spot')
+    .whereNotNull('contents')
+    .select('id', 'name', 'contents')
+  let renamed = 0
+  for (const release of releases) {
+    const entries: any[] = Array.isArray(release.contents) ? release.contents : []
+    const fromHidden = entries.some((e) => isHidden(e.name) && fileStem(String(e.name).split(/[\\/]/).pop() ?? '') === release.name)
+    if (!fromHidden) continue
+    const better = releaseNameFromContents({ entries })
+    if (!better || better === release.name) continue
+    await db.from('releases').where('id', release.id).update({ name: better })
+    renamed++
+  }
+  return renamed
+}
+
 /**
  * Renames random-named releases from their stored archive listing (no
  * downloads). Used by `release:rename`.

app/services/ArchiveWalkService.ts

@@ -0,0 +1,160 @@
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import { segmentsOf } from '#services/FileStore'
+import { looksRandom, releaseNameFromContents } from '#services/ArchiveService'
+import { rarVolume } from '#lib/content/index'
+import type { RarEntry, RarInfo } from '#lib/rar/index'
+import { probeWalk, type WalkProbe, type WalkTask } from '#lib/content/probe'
+
+/** Stored per release, as for a first-volume listing (ArchiveService). */
+const MAX_ENTRIES = 500
+
+/** Listed sizes below this share of the release's size: the first segment listed only part of it. */
+const COVERED = 0.7
+
+/** Inspection queues a walk this long after the first volume, when the release has had time to complete. */
+const WALK_DELAY_MS = 30 * 60 * 1000
+
+export interface WalkOutcome {
+  releaseId: number
+  /** Nothing downloaded: why. */
+  skipped?: 'missing' | 'covered' | 'volumes'
+  entries?: number
+  before?: number
+  complete?: boolean
+  stopped?: string
+  segments?: number
+  renamed?: string
+}
+
+/**
+ * Complete archive listings by walking RAR headers (CONCEPT.md D40). The
+ * first segment of a first volume lists the entries that start in it: for
+ * a set of many small files (an audiobook's MP3s, a music album) that is
+ * the first one or two. The walk reads one segment per further entry and
+ * volume, at low priority: its own queue, one job at a time per collection
+ * worker, and NNTP connections only when inspection isn't waiting for one.
+ */
+class ArchiveWalkService {
+  /** Segments one walk may download (~0.7 MB each). */
+  readonly maxSegments = env.get('ARCHIVE_WALK_MAX_SEGMENTS', 400)
+
+  /** A listing that leaves most of the release unaccounted for. */
+  worthWalking(entries: number, listedBytes: number, releaseSize: number): boolean {
+    if (entries === 0 || entries >= MAX_ENTRIES) return false
+    return listedBytes < COVERED * releaseSize
+  }
+
+  /** After inspection listed a first RAR volume whose listing ends inside the set. */
+  async queueAfterInspection(releaseId: number | null, info: RarInfo) {
+    if (!releaseId || info.format === '7z' || info.firstVolume === false) return
+    if (!info.truncated || info.headersEncrypted || info.entries.length === 0) return
+    await QueueService.archiveWalkQueue.add('walk', { releaseId }, { jobId: `walk-${releaseId}`, delay: WALK_DELAY_MS })
+  }
+
+  /**
+   * The database half before a walk (D41): the release's volumes, or the
+   * outcome when there's nothing to walk.
+   */
+  async prepare(releaseId: number): Promise<{ task: WalkTask } | { outcome: WalkOutcome }> {
+    const release = await db.from('releases').where('id', releaseId).select('id', 'size', 'contents').first()
+    if (!release) return { outcome: { releaseId, skipped: 'missing' } }
+    const before = Array.isArray(release.contents) ? release.contents : []
+    const listed = before.reduce((sum: number, e: any) => sum + Number(e.size ?? 0), 0)
+    if (!this.worthWalking(before.length, listed, Number(release.size))) {
+      return { outcome: await this.done(releaseId, { releaseId, skipped: 'covered' }) }
+    }
+    const volumes = await this.volumes(releaseId)
+    if (!volumes) return { outcome: await this.done(releaseId, { releaseId, skipped: 'volumes' }) }
+    return {
+      task: {
+        kind: 'walk',
+        releaseId,
+        volumes: volumes.map((file) => segmentsOf(file).map(({ number, id }) => ({ number, id }))),
+        maxSegments: this.maxSegments,
+      },
+    }
+  }
+
+  /** The database half after a walk: the fuller listing, the password flag and a better name. */
+  async apply(releaseId: number, walked: WalkProbe): Promise<WalkOutcome> {
+    const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'contents', 'encrypted').first()
+    if (!release) return { releaseId, skipped: 'missing' }
+    const before = Array.isArray(release.contents) ? release.contents : []
+    const outcome: WalkOutcome = {
+      releaseId,
+      before: before.length,
+      entries: walked.entries.length,
+      complete: walked.complete,
+      stopped: walked.stopped,
+      segments: walked.segments,
+    }
+    if (walked.entries.length > before.length) {
+      const contents = walked.entries.slice(0, MAX_ENTRIES)
+      const update: Record<string, unknown> = {
+        contents: JSON.stringify(contents),
+        encrypted: Boolean(release.encrypted) || walked.headersEncrypted || walked.entries.some((e) => e.encrypted),
+      }
+      const better = releaseNameFromContents({ entries: walked.entries as RarEntry[] })
+      // The rule of ArchiveService.record: random and post-subject names give way, a spot's title stays.
+      if (better && better !== release.name && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post')) {
+        update.name = better
+        outcome.renamed = better
+      }
+      await db.from('releases').where('id', releaseId).update(update)
+    }
+    return this.done(releaseId, outcome)
+  }
+
+  /** Walks a release here, on the inspection pool's connections when inspection isn't waiting for one. */
+  async walk(releaseId: number): Promise<WalkOutcome> {
+    const prepared = await this.prepare(releaseId)
+    if ('outcome' in prepared) return prepared.outcome
+    return this.apply(releaseId, await probeWalk(prepared.task, fetchLow))
+  }
+
+  private async done(releaseId: number, outcome: WalkOutcome) {
+    await db.from('releases').where('id', releaseId).update({ contents_walked_at: new Date() })
+    return outcome
+  }
+
+  /**
+   * The release's RAR volumes in order, from volume 1 up to the first gap.
+   * Null without a first volume (e.g. volumes with random names).
+   */
+  private async volumes(releaseId: number): Promise<any[] | null> {
+    const files = await db.from('files').where('release_id', releaseId).select('id', 'filename', 'real_name', 'segments')
+    const sets = new Map<string, Map<number, any>>()
+    for (const file of files) {
+      const volume = rarVolume(String(file.real_name ?? file.filename))
+      if (!volume || segmentsOf(file).length === 0) continue
+      const set = sets.get(volume.base) ?? new Map()
+      if (!set.has(volume.number)) set.set(volume.number, file)
+      sets.set(volume.base, set)
+    }
+    // A release holds one set, now and then a second (a sample, subs): the biggest.
+    const set = [...sets.values()].sort((a, b) => b.size - a.size)[0]
+    if (!set?.has(1)) return null
+    const ordered: any[] = []
+    for (let n = 1; set.has(n); n++) ordered.push(set.get(n))
+    return ordered
+  }
+}
+
+/** One article body on a pooled connection, behind inspection's waiting jobs. */
+async function fetchLow(messageId: string): Promise<Buffer> {
+  const conn = await NntpService.acquire(true)
+  let failed = false
+  try {
+    return await NntpService.timed(NntpService.fetchBody(conn, messageId), 'BODY')
+  } catch (error) {
+    failed = true
+    throw error
+  } finally {
+    NntpService.release(conn, failed)
+  }
+}
+
+export default new ArchiveWalkService()

app/services/BinaryFileService.ts

@@ -0,0 +1,144 @@
+import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
+import ContentService from '#services/ContentService'
+import InspectionQueue from '#services/InspectionQueue'
+
+export interface BinaryFilesResult {
+  /** New files. */
+  saved: number
+  /** Binaries whose file already existed (same first segment or file_key). */
+  existing: number
+  /** Conflicted, gone or already linked: nothing to do. */
+  skipped: number
+  /** Not all parts stored yet (the job is retried for these). */
+  incomplete: number[]
+  inspections: number
+}
+
+/**
+ * Turns completed binaries into files (worker:file), a batch at a time
+ * (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and
+ * files that exist already, one INSERT for the new files (so the files
+ * statement trigger updates each release once), and one UPDATE and DELETE
+ * for the binaries' links and copied parts. It was one job of ~8 queries per
+ * binary, ~700 a minute.
+ */
+class BinaryFileService {
+  async store(binaryIds: number[]): Promise<BinaryFilesResult> {
+    const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 }
+    const ids = [...new Set(binaryIds)]
+    const binaries = await db
+      .from('binaries')
+      .whereIn('id', ids)
+      .whereNull('conflicted_at')
+      .whereNull('file_id')
+      .select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts')
+    result.skipped = ids.length - binaries.length
+    if (binaries.length === 0) return result
+
+    const partsOf = new Map<number, { number: number; id: string; size: number }[]>()
+    const parts = await db
+      .from('parts')
+      .whereIn('binary_id', binaries.map((b) => b.id))
+      .orderBy([{ column: 'binary_id' }, { column: 'number' }])
+      .select('binary_id', 'number', 'message_id', 'bytes')
+    for (const part of parts) {
+      const list = partsOf.get(Number(part.binary_id)) ?? []
+      list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) })
+      partsOf.set(Number(part.binary_id), list)
+    }
+    const complete = binaries.filter((b) => {
+      const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts
+      if (!ok) result.incomplete.push(Number(b.id))
+      return ok
+    })
+    if (complete.length === 0) return result
+
+    // Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list).
+    const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id)
+    const byFirst = new Map<string, number>()
+    for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) {
+      byFirst.set(row.first_message_id, Number(row.id))
+    }
+    const fileIdOf = new Map<number, number>()
+    const fresh = complete.filter((b) => {
+      const existing = byFirst.get(partsOf.get(Number(b.id))![0].id)
+      if (existing) fileIdOf.set(Number(b.id), existing)
+      return !existing
+    })
+
+    const now = new Date()
+    // file_key is unique: a second completion of the same upload inserts nothing.
+    const inserted = await FileStore.insert(
+      fresh.map((b) => {
+        const segments = partsOf.get(Number(b.id))!
+        return {
+          file_key: b.file_key,
+          filename: b.name,
+          poster: b.poster,
+          date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(),
+          parts: segments.length,
+          segments,
+          groups: JSON.stringify(b.groups),
+          release_id: b.release_id,
+          real_name: b.real_name,
+          size: segments.reduce((sum, s) => sum + s.size, 0),
+          created_at: now,
+          updated_at: now,
+        }
+      })
+    )
+    const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id]))
+    const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key)
+    const byKey = new Map<string, number>()
+    if (missing.length > 0) {
+      for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id))
+    }
+    for (const b of fresh) {
+      const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
+      if (id) fileIdOf.set(Number(b.id), id)
+    }
+    const fresher = new Map(fresh.map((b) => [b.file_key, b]))
+    result.saved = inserted.length
+    result.existing = fileIdOf.size - inserted.length
+
+    // Links, locked in file_key order like ingest's upsert, so the two can't deadlock.
+    const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key))
+    if (linked.length > 0) {
+      await db.transaction(async (trx) => {
+        await trx.rawQuery(
+          `SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
+          [linked.map((b) => b.id)]
+        )
+        // Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16).
+        const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id))
+        await trx.rawQuery(
+          `UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END
+           FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`,
+          [linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))]
+        )
+      })
+      // Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
+      await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)])
+    }
+
+    // As for one file before: an SFV posted earlier may list them (D34), then inspection.
+    const added = inserted.flatMap((row) => {
+      const b = row.file_key ? fresher.get(row.file_key) : undefined
+      return b
+        ? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }]
+        : []
+    })
+    if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file))
+    // By the posted name, as before: inspection skips later volumes and readable media names anyway.
+    const jobs = added
+      .filter((file) => ContentService.isCandidate(file.posted))
+      .map((file) => file.id)
+    if (jobs.length > 0) await InspectionQueue.add(jobs)
+    result.inspections = jobs.length
+    return result
+  }
+}
+
+export default new BinaryFileService()

app/services/ContentService.ts

@@ -1,33 +1,31 @@
-import { createHash } from 'node:crypto'
 import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
 import NntpService from '#services/NntpService'
 import Par2Service, { type Par2Result } from '#services/Par2Service'
-import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
+import ArchiveService, { looksRandom, type ArchiveResult } from '#services/ArchiveService'
 import NameService from '#services/NameService'
-import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
+import { decodeSegment } from '#lib/yenc/index'
 import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
 import {
   isArchiveStart,
   isLaterVolume,
   NICOVIDEO_FILE,
   NICOVIDEO_RELEASE,
-  nicovideoInfo,
   nicovideoReleaseName,
-  sniffContentType,
-  zstdPrefix,
   type ContentType,
   type NicovideoInfo,
 } from '#lib/content/index'
-import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
+import { chunkMd5, probeInspection, type InspectProbe, type InspectTask, type TaskSegment } from '#lib/content/probe'
+import type { Segment } from '#lib/nzb/index'
 import { fileStem, releaseIdentity } from '#lib/release/index'
 import IngestService from '#services/IngestService'
 import { looksLikeFilename } from '#lib/subject/index'
 import { sevenZipVolume } from '#lib/sevenzip/index'
 import SevenZipService from '#services/SevenZipService'
 import SfvService from '#services/SfvService'
-import { segmentsOf } from '#services/FileStore'
-
-const FIRST_CHUNK = 16 * 1024
+import ArchiveWalkService from '#services/ArchiveWalkService'
+import FileStore, { segmentsOf } from '#services/FileStore'
+import GarbageFilter from '#services/GarbageFilter'
 
 /**
  * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
@@ -38,11 +36,6 @@ export function firstChunkMd5(body: Buffer, fileSize: number): string | undefine
   return chunkMd5(decodeSegment(body).data, fileSize)
 }
 
-function chunkMd5(data: Buffer, fileSize: number): string | undefined {
-  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
-  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
-}
-
 export interface ContentResult {
   par2?: Par2Result
   archive?: ArchiveResult
@@ -57,6 +50,8 @@ export interface ContentResult {
   nicovideo?: string
   /** The real name from the yEnc header, for a file posted under a random subject (D36). */
   yencName?: string
+  /** Why the file's release was deleted as garbage after this probe (D42). */
+  garbage?: string
 }
 
 /**
@@ -65,6 +60,8 @@ export interface ContentResult {
  * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
  * - par2 files are recognised by content and name every file of their set,
  * - RAR headers give the release grouping, name and contents.
+ * Inspection is prepare (database) → probe (downloads and parsing,
+ * lib/content/probe; here or on a seeding node, D41) → apply (database).
  */
 class ContentService {
   /** Random or missing names, and named first RAR volumes (for contents and password flag). */
@@ -127,89 +124,73 @@ class ContentService {
     return name
   }
 
-  /** Downloads a whole (small) file whose first segment is already decoded. */
-  private async readWhole(first: Buffer, file: any): Promise<Buffer | null> {
-    const parts = segmentsOf(file)
-    const bytes = parts.reduce((sum, p) => sum + (p.size ?? 0), 0)
-    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
-    if (parts.length === 1) return first
-
-    let assembled: YencFile | undefined
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      for (const part of parts) {
-        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
-        assembled ??= YencFile.fromSegment(segment)
-        assembled.add(segment)
-      }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
+  /**
+   * The database half before a probe (CONCEPT.md D41): what to inspect for
+   * a file, or null when there's nothing to do. A later 7z volume becomes its
+   * archive's first volume, once all volumes are in.
+   */
+  async prepare(fileId: number): Promise<InspectTask | null> {
+    const file = await db.from('files').where('id', fileId).first()
+    if (!file || !this.isCandidate(file.filename)) return null
+    // Stored before its signature was found to be garbage (D42): not worth an article.
+    if (await GarbageFilter.skipProbe(file)) return null
+    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
+      const volumes = await SevenZipService.volumes(file)
+      if (!volumes || (await FileStore.details(Number(volumes[0].id))).archive) return null
+      return this.prepare(Number(volumes[0].id))
+    }
+    const segments = segmentsOf(file)
+    if (!segments[0]?.id) return null
+    // A 7z listing reads the last volume's tail (sniffing decides whether it's 7z: random names too).
+    const volumes = await SevenZipService.volumes(file)
+    const last = volumes && volumes.length > 1 ? segmentsOf(volumes[volumes.length - 1]) : segments
+    return {
+      kind: 'inspect',
+      fileId: Number(file.id),
+      name: String(file.real_name ?? file.filename),
+      segments: taskSegments(segments),
+      sevenZip: volumes ? { volumes: volumes.length, last: taskSegments(last, true) } : null,
     }
-    return assembled!.getBuffer()
   }
 
   /**
-   * Downloads a small archive that lists NZBs and imports each of them
-   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
-   * uploads by other posters.
+   * The database half after a probe: names, groupings, listings and flags from
+   * what it found. Then the release is deleted if it turned out to be garbage (D42).
    */
-  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
-    const archive = await this.readWhole(first, file)
-    const info = archive && parseRarHeaders(archive)
-    if (!archive || !info) return []
-    const entries = info.entries.filter(isNzbEntry)
-    const results: NzbImportResult[] = []
-    for (const entry of entries) {
-      const xml = await extractEntry(archive, entry)
-      if (!xml) continue
-      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
-        entryName: entry.name,
-        moveContainer: entries.length === 1,
-      })
-      if (imported) results.push(imported)
+  async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
+    let result: ContentResult | undefined
+    // Another probe deleted a release this one writes to as garbage meanwhile (LOG.md 2026-10-08 (7)):
+    // the file's own release (nothing left to do), or one its par2 set joins. Again once that
+    // deletion has committed: the set then finds the release's files gone and makes its own.
+    for (const wait of [0, 1000, 3000]) {
+      if (wait) await new Promise((resolve) => setTimeout(resolve, wait))
+      try {
+        result = await this.applyProbe(task, probe)
+        break
+      } catch (error: any) {
+        if (error.code !== '23503' || wait === 3000) throw error
+        if (!(await db.from('files').where('id', task.fileId).select('id').first())) {
+          return { named: 0, contentType: probe.contentType, garbage: 'deleted meanwhile' }
+        }
+      }
     }
-    return results
+    if (!result) throw new Error(`Applying the probe of file ${task.fileId} gave no result`)
+    result.garbage = (await GarbageFilter.afterProbe(task.fileId, probe.contentType)) ?? undefined
+    return result
   }
 
-  async inspect(fileId: number): Promise<ContentResult | null> {
-    const file = await db.from('files').where('id', fileId).first()
-    const binary = await db.from('binaries').where('file_id', fileId).first()
-    if (!file || !this.isCandidate(file.filename)) return null
-    // A later 7z volume: list the set from its first volume once all volumes are in.
-    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
-      const volumes = await SevenZipService.volumes(file)
-      if (!volumes || volumes[0].archive) return null
-      return this.inspect(Number(volumes[0].id))
+  private async applyProbe(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
+    const result: ContentResult = { named: 0, contentType: probe.contentType }
+    const file = await db.from('files').where('id', task.fileId).first()
+    if (!file) return result
+    const binary = await db.from('binaries').where('file_id', task.fileId).first()
+    const { contentType } = probe
+    await FileStore.setDetails(Number(file.id), { content_type: contentType })
+    if (contentType === 'missing') {
+      // Nothing to read; the binary needn't wait for inspection any more.
+      if (binary) await db.from('binaries').where('id', binary.id).update({ awaiting_inspection: null })
+      return result
     }
-    const first = segmentsOf(file)[0]
-    if (!first?.id) return null
-
-    const conn = await NntpService.acquire()
-    let failed = false
-    let body: Buffer
-    try {
-      body = await NntpService.fetchBody(conn, first.id)
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
-    let segment
-    try {
-      segment = decodeSegment(body)
-    } catch (error) {
-      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
-      segment = decodeSegment(body, { lenientSize: true })
-    }
-    const size = segment.meta.begin.size
-    const contentType = sniffContentType(segment.data)
-    await db.from('files').where('id', fileId).update({ content_type: contentType })
 
     // Fingerprint, so par2 sets (now or later) can name this file exactly.
     if (binary) {
@@ -217,13 +198,14 @@ class ContentService {
         .from('binaries')
         .where('id', binary.id)
         .update({
-          size: db.raw('coalesce(size, ?)', [size]),
-          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
+          size: db.raw('coalesce(size, ?)', [probe.size]),
+          md5_16k: db.raw('coalesce(md5_16k, ?)', [probe.md5_16k]),
+          // Inspected: the GC may delete it again by its usual rules (LOG.md 2026-10-06 (11)).
+          awaiting_inspection: null,
         })
     }
 
-    const result: ContentResult = { named: 0, contentType }
-    result.yencName = await this.nameFromYenc(file, binary, segment.meta.begin.name)
+    result.yencName = await this.nameFromYenc(file, binary, probe.yencName ?? '')
     if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
       // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
       // to read, and useless without the key.
@@ -233,51 +215,107 @@ class ContentService {
       }
       return result
     }
-    const nicovideo = NICOVIDEO_FILE.exec(String(file.real_name ?? file.filename))
+    const nicovideo = NICOVIDEO_FILE.exec(task.name)
     if (nicovideo) {
-      const info = nicovideoInfo(await zstdPrefix(segment.data))
-      if (info) result.nicovideo = await this.nameNicovideo(file, info, nicovideo[1])
+      if (probe.nicovideo) result.nicovideo = await this.nameNicovideo(file, probe.nicovideo, nicovideo[1])
       return result
     }
-    if (/\.sfv$/i.test(String(file.real_name ?? file.filename))) {
-      const text = await this.readWhole(segment.data, file)
-      if (text) result.sfvMoved = (await SfvService.attach(file, text.toString('latin1'))) ?? undefined
+    if (/\.sfv$/i.test(task.name)) {
+      if (typeof probe.text === 'string') result.sfvMoved = (await SfvService.attach(file, probe.text)) ?? undefined
       return result
     }
     if (contentType === 'nzb') {
-      const xml = await this.readWhole(segment.data, file)
-      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
+      if (typeof probe.text === 'string') result.nzb = (await NzbImportService.import(file, probe.text)) ?? undefined
       return result
     }
-    const set = await Par2Service.read(segment.data, segmentsOf(file))
-    if (set && binary) {
-      result.par2 = await Par2Service.attach(binary, set)
-    } else if (contentType === '7z') {
-      const info = await SevenZipService.list(file, segment.data)
-      if (info) result.archive = await ArchiveService.record(file, binary, info)
-    } else {
-      const info = parseRarHeaders(segment.data)
-      if (info) {
-        result.archive = await ArchiveService.record(file, binary, info)
-        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
+    if (probe.par2 && binary) {
+      result.par2 = await Par2Service.attach(binary, probe.par2)
+    } else if (probe.archive && probe.archive.format === '7z') {
+      result.archive = await ArchiveService.record(file, binary, probe.archive)
+    } else if (probe.archive) {
+      result.archive = await ArchiveService.record(file, binary, probe.archive)
+      // The rest of the listing, later and at low priority (D40).
+      await ArchiveWalkService.queueAfterInspection(result.archive.releaseId ?? null, probe.archive)
+      if (probe.archiveNzbs?.length) {
+        result.archiveNzbs = []
+        for (const { entryName, xml } of probe.archiveNzbs) {
+          const imported = await NzbImportService.import(file, xml, { entryName, moveContainer: (probe.archiveNzbEntries ?? 0) === 1 })
+          if (imported) result.archiveNzbs.push(imported)
+        }
       }
     }
     if (binary) result.named = await NameService.resolve([binary.id])
     return result
   }
+
+  /** Inspects a file here: prepare, probe on the inspection pool's connections, apply. */
+  async inspect(fileId: number): Promise<ContentResult | null> {
+    const task = await this.prepare(fileId)
+    if (!task) return null
+    const probe = await probeInspection(task, fetchPooled, (message) => logger.warn(message))
+    return this.apply(task, probe)
+  }
 }
 
-/** NZBs that can be read from a single, unencrypted archive. */
-function hasNzbEntries(info: RarInfo): boolean {
-  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+/** A task's segment list: all of a small file's; of a big one, the first and the last few (a 7z tail). */
+function taskSegments(segments: Segment[], tailOnly = false): TaskSegment[] {
+  const list = segments.map(({ number, id, size }) => ({ number, id, size }))
+  if (list.length <= TASK_SEGMENTS) return list
+  return tailOnly ? list.slice(-TASK_SEGMENTS) : [list[0], ...list.slice(-TASK_SEGMENTS)]
 }
 
-function isNzbEntry(entry: RarEntry): boolean {
-  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+/** More than whole files (40 parts) and par2 files (20) are read; a 7z tail of 16 MB fits easily. */
+const TASK_SEGMENTS = 40
+
+/** One article body on a pooled connection. */
+async function fetchPooled(messageId: string): Promise<Buffer> {
+  const conn = await NntpService.acquire()
+  let failed = false
+  try {
+    return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
+  } catch (error: any) {
+    // A missing article (430) leaves the connection fine.
+    failed = !/^4\d\d/.test(String(error.message))
+    throw error
+  } finally {
+    NntpService.release(conn, failed)
+  }
 }
 
-/** Small posted files (NZBs) that are read completely. */
-const WHOLE_MAX_PARTS = 40
-const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+/**
+ * One line per finding, as worker:collection logs them (and the seed API
+ * for nodes' results).
+ */
+export function describeContent(filename: string, content: ContentResult | null): string[] {
+  const lines: string[] = []
+  if (content?.par2) {
+    const { par2 } = content
+    lines.push(`Par2 "${filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
+  } else if (content?.archive) {
+    const { info, releaseRenamed, grouped } = content.archive
+    const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
+      .filter(Boolean)
+      .join(', ')
+    lines.push(
+      `Archive "${filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
+        (grouped ? ', grouped into a release' : '') +
+        (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
+    )
+  }
+  if (content?.named) lines.push(`"${filename}" got its real name from par2.`)
+  if (content?.contentType === 'missing') lines.push(`"${filename}": the server no longer has its first article.`)
+  if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
+  if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
+  if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
+  if (content?.garbage) lines.push(`"${filename}": release deleted as garbage (${content.garbage === 'deleted meanwhile' ? 'by another probe' : `${content.garbage}, no name or password`}).`)
+  if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
+  if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
+  if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)
+  if (content?.nzb) {
+    const n = content.nzb
+    lines.push(`NZB "${filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
+  }
+  return lines
+}
 
 export default new ContentService()

app/services/FileKeys.ts

@@ -0,0 +1,60 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** files ids per slice: a few seconds of reads from the HDD. */
+const SLICE = 100_000
+
+/**
+ * The file_keys filter (LOG.md 2026-10-06 (7)): a 64-bit hash of every
+ * file_key. The insert trigger adds new keys; `fill` copies the keys of
+ * files from before, in id order and a slice at a time, and marks the
+ * filter complete when it reaches the files that existed when it started
+ * (everything after that has been added by the trigger, which was live
+ * first). FileStore uses the filter only once it is complete.
+ */
+class FileKeys {
+  private completeSince = 0
+
+  /** Whether every file's key is in file_keys; cached once true. */
+  async complete(): Promise<boolean> {
+    if (this.completeSince) return true
+    const row = await db.from('file_keys_fill').where('id', 1).first()
+    if (row?.complete) this.completeSince = Date.now()
+    return Boolean(row?.complete)
+  }
+
+  /** Forgets the cached state (tests). */
+  forget() {
+    this.completeSince = 0
+  }
+
+  /** Copies keys until `budgetMs` has passed or the fill is done; returns the files read and whether it's complete. */
+  async fill(budgetMs: number): Promise<{ read: number; complete: boolean; nextId: number; untilId: number }> {
+    const started = Date.now()
+    let state = await db.from('file_keys_fill').where('id', 1).first()
+    if (!state) return { read: 0, complete: false, nextId: 0, untilId: 0 }
+    if (state.until_id === null) {
+      // Everything inserted from here on goes through the trigger (live since the migration).
+      const [{ max }] = await db.from('files').max('id as max')
+      await db.from('file_keys_fill').where('id', 1).update({ until_id: Number(max ?? 0), updated_at: new Date() })
+      state = await db.from('file_keys_fill').where('id', 1).first()
+    }
+    let next = Number(state.next_id)
+    const until = Number(state.until_id)
+    let read = 0
+    while (next <= until && Date.now() - started < budgetMs) {
+      const to = Math.min(until, next + SLICE - 1)
+      const result = await db.rawQuery(
+        `INSERT INTO file_keys (key_hash)
+         SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE id BETWEEN ? AND ? AND file_key IS NOT NULL
+         ON CONFLICT DO NOTHING`,
+        [next, to]
+      )
+      read += Number(result.rowCount ?? 0)
+      next = to + 1
+      await db.from('file_keys_fill').where('id', 1).update({ next_id: next, complete: next > until, updated_at: new Date() })
+    }
+    return { read, complete: next > until, nextId: next, untilId: until }
+  }
+}
+
+export default new FileKeys()

app/services/FilePartitions.ts

@@ -3,13 +3,16 @@ import logger from '@adonisjs/core/services/logger'
 
 /** Ids per partition (as the migration that partitioned files: …150). */
 export const FILE_PARTITION_SIZE = 4_000_000
-/** Ids kept covered beyond the newest file: two partitions. */
-const AHEAD = 2 * FILE_PARTITION_SIZE
+/**
+ * Ids kept covered beyond the newest file: four partitions (16M ids, about a day of
+ * seeding), so a few sweeps that can't create one are harmless.
+ */
+const AHEAD = 4 * FILE_PARTITION_SIZE
 
 /**
  * `files` is partitioned by id (CONCEPT.md D39). An insert with an id no
  * partition covers fails, so gc:scheduler calls ensure() every sweep to keep
- * two partitions ready ahead of the newest id. Each new partition gets its
+ * four partitions ready ahead of the newest id. Each new partition gets its
  * own unique index on file_key (FileStore.insert checks the older ones),
  * in the parent's tablespace (the HDD on homelab).
  */
@@ -17,8 +20,9 @@ class FilePartitions {
   async ensure(): Promise<string[]> {
     const created: string[] = []
     while (true) {
+      // The sequence, not max(id): cheaper, and it counts ids handed out but not committed yet.
       const { rows } = await db.rawQuery(
-        `SELECT (SELECT coalesce(max(id), 0) FROM files) AS max_id,
+        `SELECT (SELECT last_value FROM files_id_seq) AS max_id,
                 -- ''{0,1}, not ''?: knex reads ? as a binding.
                 max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
          FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
@@ -28,8 +32,15 @@ class FilePartitions {
       const upper = Number(rows[0].upper)
       if (upper - maxId >= AHEAD) return created
       const name = `files_p${upper}`
-      await db.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`)
-      await db.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
+      // CREATE … PARTITION OF locks files and its partitions one by one, which deadlocked with
+      // file inserts (11 failed sweeps on 2026-10-07). Taking the parent's lock first, in one
+      // statement, can only wait; after 10 s it gives up until the next sweep.
+      await db.transaction(async (trx) => {
+        await trx.rawQuery(`SET LOCAL lock_timeout = '10s'`)
+        await trx.rawQuery(`LOCK TABLE files IN ACCESS EXCLUSIVE MODE`)
+        await trx.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`)
+        await trx.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
+      })
       logger.info(`files: partition ${name} created (ids ${upper}–${upper + FILE_PARTITION_SIZE - 1}).`)
       created.push(name)
     }

app/services/FileStore.ts

@@ -1,4 +1,5 @@
 import db from '@adonisjs/lucid/services/db'
+import FileKeys from '#services/FileKeys'
 import type { QueryClientContract } from '@adonisjs/lucid/types/database'
 import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
 
@@ -36,20 +37,66 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
   return stored ? unpackSegments(stored) : []
 }
 
+/** What inspection learns about a file, kept beside `files` (LOG.md 2026-10-08 (14)). */
+export interface FileDetails {
+  content_type?: string | null
+  archive?: unknown
+}
+
+const DETAIL_COLUMNS = ['content_type', 'archive'] as const
+
+/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
+export const insertLimits = { rows: 2000 }
+
 class FileStore {
+  /** Sets some of a file's details, leaving the others as they are. */
+  async setDetails(fileId: number, details: FileDetails, client: Pick<QueryClientContract, 'rawQuery'> = db) {
+    const columns = DETAIL_COLUMNS.filter((c) => details[c] !== undefined)
+    if (columns.length === 0) return
+    const values = columns.map((c) => (c === 'archive' && details[c] !== null ? JSON.stringify(details[c]) : details[c]))
+    await client.rawQuery(
+      `INSERT INTO file_details (file_id, ${columns.join(', ')}) VALUES (?, ${columns.map(() => '?').join(', ')})
+       ON CONFLICT (file_id) DO UPDATE SET ${columns.map((c) => `${c} = excluded.${c}`).join(', ')}`,
+      [fileId, ...(values as any[])]
+    )
+  }
+
+  /** A file's details (empty when it has none). */
+  async details(fileId: number): Promise<FileDetails> {
+    return (await db.from('file_details').where('file_id', fileId).select('content_type', 'archive').first()) ?? {}
+  }
+
   /**
    * Inserts files, skipping those whose `file_key` exists. Returns the rows
-   * that were inserted (id and filename). `files` is partitioned by id
+   * that were inserted (id, filename, file_key). `files` is partitioned by id
    * (CONCEPT.md D39) and each partition has its own unique file_key index:
-   * keys already in any partition are looked up first; two writers adding
-   * the same new key at once meet in the newest partition's index.
+   * once file_keys (a 64-bit hash of every key) is complete, it says which keys
+   * may exist, and only those are looked up in the partitions; two
+   * writers adding the same new key at once meet in the newest partition's index.
    */
-  async insert(files: NewFile[], client: Pick<QueryClientContract, 'knexQuery'> = db): Promise<{ id: number; filename: string }[]> {
+  async insert(
+    files: NewFile[],
+    client: Pick<QueryClientContract, 'knexQuery' | 'rawQuery'> = db
+  ): Promise<{ id: number; filename: string; file_key: string | null }[]> {
     if (files.length === 0) return []
     const keys = [...new Set(files.map((file) => file.file_key).filter((key): key is string => typeof key === 'string'))]
     const known = new Set<string>()
+    const filtered = await FileKeys.complete()
     for (let i = 0; i < keys.length; i += 5000) {
-      const found = await client.knexQuery().from('files').whereIn('file_key', keys.slice(i, i + 5000)).select('file_key')
+      const chunk = keys.slice(i, i + 5000)
+      if (!filtered) {
+        const found = await client.knexQuery().from('files').whereIn('file_key', chunk).select('file_key')
+        for (const row of found) known.add(row.file_key)
+        continue
+      }
+      const { rows } = await client.rawQuery(
+        `SELECT k.key FROM unnest(?::text[]) AS k(key)
+         WHERE EXISTS (SELECT 1 FROM file_keys f WHERE f.key_hash = hashtextextended(k.key, 0))`,
+        [chunk]
+      )
+      const maybe = rows.map((row: { key: string }) => row.key)
+      if (maybe.length === 0) continue
+      const found = await client.knexQuery().from('files').whereIn('file_key', maybe).select('file_key')
       for (const row of found) known.add(row.file_key)
     }
     const seen = new Set<string>()
@@ -66,8 +113,25 @@ class FileStore {
         return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
       })
     if (rows.length === 0) return []
-    const inserted = await client.knexQuery().table('files').insert(rows).onConflict().ignore().returning(['id', 'filename'])
-    return inserted.map((row: any) => ({ id: Number(row.id), filename: row.filename }))
+    // Details given with a new file (e.g. a known content type) go to file_details.
+    const details = rows.map((row: Record<string, unknown>) => {
+      const d: FileDetails = {}
+      for (const c of DETAIL_COLUMNS) if (c in row) (d as any)[c] = row[c]
+      for (const c of DETAIL_COLUMNS) delete row[c]
+      return d
+    })
+    // A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
+    const inserted: { id: number; filename: string; file_key: string | null }[] = []
+    for (let i = 0; i < rows.length; i += insertLimits.rows) {
+      const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename', 'file_key'])
+      for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename, file_key: row.file_key ?? null })
+    }
+    const detailsByKey = new Map(rows.map((row: Record<string, unknown>, i: number) => [row.file_key ?? row.filename, details[i]]))
+    for (const row of inserted) {
+      const d = detailsByKey.get(row.file_key ?? row.filename)
+      if (d && Object.keys(d).length > 0) await this.setDetails(row.id, d, client)
+    }
+    return inserted
   }
 }
 

app/services/GarbageFilter.ts

@@ -0,0 +1,217 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import { looksRandom } from '#services/ArchiveService'
+import { fileStem } from '#lib/release/index'
+import { explored, signatureGroup, signatureKey, uploadTool } from '#lib/garbage/index'
+
+/** What the filter needs to know about an upload before it is stored. */
+export interface Upload {
+  /** The posted file name. */
+  name: string
+  /** Any of its articles' Message-IDs (one tool posts all of a file). */
+  messageId: string | null | undefined
+  groups: unknown
+  /** Its release's key (the file key when it has none): kept or dropped as a whole. */
+  releaseKey: string
+}
+
+/** Contents that can't name a random-named release without a key (LOG.md 2026-10-08 (5)). */
+const CONTAINERS = ['rclone', 'nzbe', 'yencbin']
+
+/** How long the dropped signatures are cached per process. */
+const CACHE_MS = 5 * 60 * 1000
+
+export function isRandomName(name: string) {
+  return looksRandom(fileStem(name))
+}
+
+/**
+ * Garbage: random-named uploads that never get a real name and have no
+ * known password (CONCEPT.md D42). Two filters:
+ * - before storing (seeding, ingest): random-named uploads whose signature
+ *   (posting tool, first group) is dropped, or whose release was dropped
+ *   after its probe, aren't stored;
+ * - after a probe: a random-named release whose probes are all done, that
+ *   is still without a real name, password, names from its files or spot,
+ *   and is encrypted, a container or missing its first article, is deleted.
+ * The dropped signatures are learned from a sample of probed releases
+ * (learn(), gc:scheduler).
+ */
+class GarbageFilter {
+  private dropped: Set<string> | null = null
+  private loadedAt = 0
+
+  /** The dropped signatures, cached for CACHE_MS. */
+  async droppedSignatures(): Promise<Set<string>> {
+    if (this.dropped && Date.now() - this.loadedAt < CACHE_MS) return this.dropped
+    const rows = await db.from('upload_signatures').where('dropped', true).select('tool', 'group_name')
+    this.dropped = new Set(rows.map((row) => signatureKey(row.tool, row.group_name)))
+    this.loadedAt = Date.now()
+    return this.dropped
+  }
+
+  /** Forgets the cached signatures (after learn(), and in tests). */
+  reset() {
+    this.dropped = null
+  }
+
+  /** Whether a random-named upload's signature is dropped and it isn't in the explored share. */
+  private droppedBySignature(upload: Upload, signatures: Set<string>) {
+    if (signatures.size === 0) return false
+    if (!signatures.has(signatureKey(uploadTool(upload.messageId), signatureGroup(upload.groups)))) return false
+    return !explored(upload.releaseKey, indexerConfig.garbage.explorePercent)
+  }
+
+  /**
+   * The items worth storing: drops random-named uploads of dropped signatures
+   * (but explorePercent of their releases) and of dropped releases.
+   */
+  async keep<T>(items: T[], upload: (item: T) => Upload): Promise<{ kept: T[]; dropped: number }> {
+    if (!indexerConfig.garbage.enabled || items.length === 0) return { kept: items, dropped: 0 }
+    const signatures = await this.droppedSignatures()
+    // Per release, kept as a whole when any of its files has a readable name. A file is
+    // looked at once: an ingest batch has many articles of each.
+    const verdicts = new Map<string, 'keep' | 'drop' | 'check'>()
+    const seen = new Set<string>()
+    for (const item of items) {
+      const u = upload(item)
+      if (verdicts.get(u.releaseKey) === 'keep' || seen.has(`${u.releaseKey}\n${u.name}`)) continue
+      seen.add(`${u.releaseKey}\n${u.name}`)
+      if (!isRandomName(u.name)) verdicts.set(u.releaseKey, 'keep')
+      else if (!verdicts.has(u.releaseKey)) verdicts.set(u.releaseKey, this.droppedBySignature(u, signatures) ? 'drop' : 'check')
+    }
+    const check = [...verdicts].filter(([, v]) => v === 'check').map(([key]) => key)
+    for (let i = 0; i < check.length; i += 5000) {
+      const rows = await db.from('dropped_releases').whereIn('release_key', check.slice(i, i + 5000)).select('release_key')
+      for (const row of rows) verdicts.set(row.release_key, 'drop')
+    }
+    const kept = items.filter((item) => verdicts.get(upload(item).releaseKey) !== 'drop')
+    return { kept, dropped: items.length - kept.length }
+  }
+
+  /**
+   * Whether a stored file's probe can be skipped: a random-named file of a
+   * dropped signature, outside the explored share (files stored before its
+   * signature was dropped). Its binary stops waiting for inspection.
+   */
+  async skipProbe(file: { id: number | string; filename: string; real_name?: string | null; first_message_id?: string | null; groups: unknown; release_id?: number | string | null }) {
+    if (!indexerConfig.garbage.enabled || file.real_name || !isRandomName(file.filename)) return false
+    const signatures = await this.droppedSignatures()
+    if (!signatures.has(signatureKey(uploadTool(file.first_message_id), signatureGroup(file.groups)))) return false
+    const release = file.release_id ? await db.from('releases').where('id', file.release_id).select('release_key').first() : null
+    if (explored(release?.release_key ?? String(file.id), indexerConfig.garbage.explorePercent)) return false
+    await db.from('binaries').where('file_id', file.id).where('awaiting_inspection', true).update({ awaiting_inspection: null })
+    return true
+  }
+
+  /**
+   * After a file's probe has been applied: notes its release for learning
+   * (a sample) and deletes the release when it is garbage. Never throws:
+   * the probe's result is stored already. Returns why the release was
+   * deleted, or null.
+   */
+  async afterProbe(fileId: number, contentType: string | null): Promise<string | null> {
+    try {
+      const file = await db.from('files').where('id', fileId).select('id', 'filename', 'real_name', 'first_message_id', 'groups', 'release_id').first()
+      if (!file?.release_id || !isRandomName(String(file.filename))) return null
+      const releaseId = Number(file.release_id)
+      if (releaseId % indexerConfig.garbage.probeSample === 0) {
+        await db.rawQuery(`INSERT INTO signature_probes (release_id, tool, group_name) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [
+          releaseId,
+          uploadTool(file.first_message_id),
+          signatureGroup(file.groups),
+        ])
+      }
+      if (!indexerConfig.garbage.enabled) return null
+      return await this.dropIfGarbage(releaseId, contentType)
+    } catch (error: any) {
+      logger.warn(`Garbage check after probing file ${fileId}: ${error.message}`)
+      return null
+    }
+  }
+
+  private async dropIfGarbage(releaseId: number, contentType: string | null): Promise<string | null> {
+    const { rows } = await db.rawQuery(
+      `SELECT r.release_key, r.container, r.encrypted,
+         is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
+           OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id) AS named,
+         EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id AND b.awaiting_inspection) AS waiting
+       FROM releases r WHERE r.id = ?`,
+      [releaseId]
+    )
+    const release = rows[0]
+    if (!release || release.named || release.waiting) return null
+    const reason = CONTAINERS.includes(release.container) ? release.container : release.encrypted === true ? 'encrypted' : contentType === 'missing' ? 'missing' : null
+    if (!reason) return null
+    await db.transaction(async (trx) => {
+      await trx.rawQuery(`INSERT INTO dropped_releases (release_key, reason) VALUES (?, ?) ON CONFLICT DO NOTHING`, [release.release_key, reason])
+      await trx.rawQuery(`DELETE FROM binaries WHERE release_id = ?`, [releaseId])
+      await trx.rawQuery(`DELETE FROM files WHERE release_id = ?`, [releaseId])
+      await trx.rawQuery(`DELETE FROM releases WHERE id = ?`, [releaseId])
+    })
+    return reason
+  }
+
+  /**
+   * Learns the signatures from the probed releases of the last windowDays:
+   * dropped when at least minProbes are known and at most maxNamedRate got a
+   * real name, password, names from their files or a spot. A signature with
+   * fewer probes keeps its verdict (a dropped one only gets explored probes).
+   * Releases deleted as garbage count as unnamed. Returns the signatures
+   * whose verdict changed.
+   */
+  async learn(): Promise<{ signatures: number; dropped: number; changed: string[] }> {
+    const { minProbes, maxNamedRate, windowDays } = indexerConfig.garbage
+    await db.rawQuery(`DELETE FROM signature_probes WHERE probed_at < now() - make_interval(days => ?)`, [windowDays])
+    const before = new Map((await db.from('upload_signatures').select('tool', 'group_name', 'dropped')).map((row) => [signatureKey(row.tool, row.group_name), row.dropped]))
+    await db.rawQuery(
+      `INSERT INTO upload_signatures (tool, group_name, probed, named, dropped, updated_at)
+       SELECT tool, group_name, probed, named, probed >= ?::int AND named <= probed * ?::float8, now()
+       FROM (
+         SELECT p.tool, p.group_name, count(*)::int AS probed,
+           (count(*) FILTER (WHERE is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
+              OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)))::int AS named
+         FROM signature_probes p LEFT JOIN releases r ON r.id = p.release_id
+         GROUP BY p.tool, p.group_name
+       ) s
+       ON CONFLICT (tool, group_name) DO UPDATE SET
+         probed = excluded.probed, named = excluded.named, updated_at = now(),
+         dropped = CASE WHEN excluded.probed >= ?::int THEN excluded.dropped ELSE upload_signatures.dropped END`,
+      [minProbes, maxNamedRate, minProbes]
+    )
+    const after = await db.from('upload_signatures').select('tool', 'group_name', 'dropped', 'probed', 'named')
+    const changed = after
+      .filter((row) => (before.get(signatureKey(row.tool, row.group_name)) ?? false) !== row.dropped)
+      .map((row) => `${row.tool} ${row.group_name || '(no group)'}: ${row.dropped ? 'dropped' : 'kept'} (${row.named}/${row.probed} named)`)
+    this.reset()
+    return { signatures: after.length, dropped: after.filter((row) => row.dropped).length, changed }
+  }
+
+  /**
+   * Fills signature_probes from a sample of files probed before it existed
+   * (garbage:learn --seed): random-named files with a content type, from
+   * TABLESAMPLE SYSTEM (percent) of files. Returns the releases added.
+   */
+  async seed(percent: number): Promise<number> {
+    const { rows } = await db.rawQuery(
+      `SELECT DISTINCT ON (f.release_id) f.release_id, f.first_message_id, f.groups
+       FROM files f TABLESAMPLE SYSTEM (?) JOIN file_details d ON d.file_id = f.id
+       WHERE d.content_type IS NOT NULL AND f.release_id IS NOT NULL AND NOT is_readable_name(f.filename)`,
+      [percent]
+    )
+    let added = 0
+    for (let i = 0; i < rows.length; i += 2000) {
+      const chunk = rows.slice(i, i + 2000)
+      const result = await db.rawQuery(
+        `INSERT INTO signature_probes (release_id, tool, group_name)
+         SELECT * FROM unnest(?::bigint[], ?::text[], ?::text[]) ON CONFLICT DO NOTHING`,
+        [chunk.map((row: any) => row.release_id), chunk.map((row: any) => uploadTool(row.first_message_id)), chunk.map((row: any) => signatureGroup(row.groups))]
+      )
+      added += result.rowCount ?? 0
+    }
+    return added
+  }
+}
+
+export default new GarbageFilter()

app/services/IngestService.ts

@@ -4,6 +4,7 @@ import type { QueryClientContract } from '@adonisjs/lucid/types/database'
 import type { Knex } from 'knex'
 import QueueService from '#services/QueueService'
 import NameService from '#services/NameService'
+import GarbageFilter from '#services/GarbageFilter'
 import type { IngestSegment } from '#lib/ingest/segment'
 
 // Header parsing is pure and lives in lib, so seeding nodes need no database (CONCEPT.md D37).
@@ -85,6 +86,9 @@ async function upsertKeyed(
     }
   }
 }
+/** Completed binaries per worker:file job. */
+const FILE_BATCH = 100
+
 /** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */
 export const PAR2_SIZE_ONLY_MIN = 1024 * 1024
 /** A par2 set and its files are posted together; don't match across days. */
@@ -107,7 +111,14 @@ function validDate(date?: Date) {
  * no-op. See CONCEPT.md (D12).
  */
 class IngestService {
-  async ingest(segments: IngestSegment[]): Promise<CompletedBinary[]> {
+  async ingest(batch: IngestSegment[]): Promise<CompletedBinary[]> {
+    // Garbage isn't stored at all: no release, binary or parts (CONCEPT.md D42).
+    const { kept: segments } = await GarbageFilter.keep(batch, (s) => ({
+      name: s.name,
+      messageId: s.messageId,
+      groups: s.groups,
+      releaseKey: s.release?.key ?? s.fileKey,
+    }))
     if (segments.length === 0) return []
 
     // Short statements, no enclosing transaction: every step is idempotent, and
@@ -187,8 +198,29 @@ class IngestService {
       const newParts = partRows.filter((row) => !(done.has(row.binary_id) && row.pcrc32 === null))
 
       newParts.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
+      // Parts and the binaries' stored-part counts in one statement, so the count can't miss a
+      // part (LOG.md 2026-10-06 (7)); claimCompleted compares it instead of counting parts. NULL
+      // (binaries from before part_count) is counted once: the statement's own rows aren't in
+      // its snapshot, so they're added on top. Concurrent batches can lock binaries in a
+      // different order: a deadlock rolls the whole statement back, and it is retried.
       for (const chunk of chunks(newParts, PART_CHUNK)) {
-        await knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore()
+        const insert = knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore().returning('binary_id').toSQL()
+        for (let attempt = 1; ; attempt++) {
+          try {
+            await knex.raw(
+              `WITH added AS (${insert.sql}),
+               counted AS (SELECT binary_id, count(*)::int AS n FROM added GROUP BY binary_id)
+               UPDATE binaries b SET part_count = CASE
+                 WHEN b.part_count IS NULL THEN (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) + c.n
+                 ELSE b.part_count + c.n END
+               FROM counted c WHERE b.id = c.binary_id`,
+              insert.bindings as any[]
+            )
+            break
+          } catch (error: any) {
+            if (error.code !== '40P01' || attempt >= 5) throw error
+          }
+        }
       }
 
       if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows)
@@ -272,18 +304,27 @@ class IngestService {
    */
   async matchPar2Releases(binaryIds: number[]) {
     if (binaryIds.length === 0) return
+    // Listed-once first (index-only on release_files (release_id, size): most candidate
+    // sizes are a set's equal-sized volumes), the release's date only for what's left;
+    // OFFSET 0 keeps that order. Was: every size match's date and count, 18 s per batch.
     await db.rawQuery(
       `UPDATE binaries b SET release_id = m.release_id
        FROM (
-         SELECT b2.id, min(rf.release_id) AS release_id
-         FROM binaries b2
-         JOIN release_files rf ON rf.size = b2.size
-         JOIN releases r ON r.id = rf.release_id
-         WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
-           AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ?
-           AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1
-         GROUP BY b2.id
-         HAVING count(DISTINCT rf.release_id) = 1
+         SELECT c.id, min(c.release_id) AS release_id
+         FROM (
+           SELECT b2.id, b2.posted_at, rf.release_id
+           FROM binaries b2
+           JOIN release_files rf ON rf.size = b2.size
+           WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
+             AND NOT EXISTS (
+               SELECT 1 FROM release_files x
+               WHERE x.release_id = rf.release_id AND x.size = rf.size AND x.par2_file_id <> rf.par2_file_id)
+           OFFSET 0
+         ) c
+         JOIN releases r ON r.id = c.release_id
+         WHERE abs(extract(epoch FROM (r.posted_at - c.posted_at))) < ?
+         GROUP BY c.id
+         HAVING count(DISTINCT c.release_id) = 1
        ) m
        WHERE b.id = m.id`,
       [binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
@@ -357,7 +398,7 @@ class IngestService {
     const result = await db.rawQuery(
       `UPDATE binaries b SET completed_at = now()
        WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL
-         AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) >= b.total_parts
+         AND coalesce(b.part_count, (SELECT count(*) FROM parts p WHERE p.binary_id = b.id)) >= b.total_parts
        RETURNING b.id, b.file_key`,
       [binaryIds]
     )
@@ -369,14 +410,14 @@ class IngestService {
 
   async enqueue(completed: CompletedBinary[]) {
     if (completed.length === 0) return
-    // jobId = file_key: BullMQ ignores an add() for an id that already exists.
-    await QueueService.fileQueue.addBulk(
-      completed.map(({ id, fileKey }) => ({
-        name: 'process-file',
-        data: { binaryId: id },
-        opts: { jobId: fileKey },
-      }))
-    )
+    // A job per FILE_BATCH binaries: worker:file stores them with a few set-based queries (LOG.md 2026-10-06 (7)).
+    // jobId from the first file_key: BullMQ ignores an add() for an id that already exists.
+    const jobs = []
+    for (let i = 0; i < completed.length; i += FILE_BATCH) {
+      const batch = completed.slice(i, i + FILE_BATCH)
+      jobs.push({ name: 'process-files', data: { binaryIds: batch.map((b) => b.id) }, opts: { jobId: `${batch[0].fileKey}-${batch.length}` } })
+    }
+    await QueueService.fileQueue.addBulk(jobs)
   }
 }
 

app/services/InspectionQueue.ts

@@ -0,0 +1,66 @@
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import QueueService from '#services/QueueService'
+
+/** Files per transaction when moving from the table to BullMQ. */
+const FEED_BATCH = 5000
+
+/** Priorities: lower goes first. */
+export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const
+
+/**
+ * The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)).
+ * Producers add file ids to the inspection_queue table; feed() moves them
+ * into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs
+ * wait there, where worker:collection and inspecting nodes take them as
+ * before. Redis holds a working set instead of a backlog of days.
+ */
+class InspectionQueue {
+  readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000)
+
+  /** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */
+  async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) {
+    const ids = [...new Set(fileIds)]
+    for (let i = 0; i < ids.length; i += 10_000) {
+      await db.rawQuery(
+        `INSERT INTO inspection_queue (file_id, priority)
+         SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id
+         ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`,
+        [priority, ids.slice(i, i + 10_000)]
+      )
+    }
+  }
+
+  /** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */
+  async feed(): Promise<number> {
+    const queue = QueueService.collectionQueue
+    const counts = await queue.getJobCounts('wait', 'prioritized')
+    let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0)
+    let moved = 0
+    while (room > 0) {
+      const n = await db.transaction(async (trx) => {
+        const { rows } = await trx.rawQuery(
+          `SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`,
+          [Math.min(room, FEED_BATCH)]
+        )
+        if (rows.length === 0) return 0
+        const ids = rows.map((row: { file_id: string }) => Number(row.file_id))
+        // Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id).
+        await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } })))
+        await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids])
+        return ids.length
+      })
+      if (n === 0) break
+      moved += n
+      room -= n
+    }
+    return moved
+  }
+
+  async size(): Promise<number> {
+    const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`)
+    return Number(rows[0].n)
+  }
+}
+
+export default new InspectionQueue()

app/services/NntpService.ts

@@ -7,6 +7,8 @@ class NntpService {
   private allConnections: Set<any>;
   private idleConnections: any[];
   private waiters: ((conn: any) => void)[];
+  /** Background work (archive walks): served only when no one else waits. */
+  private lowWaiters: ((conn: any) => void)[] = [];
   private createdCount: number;
 
   constructor(poolSize = 10) {
@@ -34,7 +36,8 @@ class NntpService {
     return conn;
   }
 
-  public async acquire() {
+  /** A connection from the pool; with `low`, after every waiting normal caller. */
+  public async acquire(low = false) {
     if (this.idleConnections.length > 0) {
       return this.idleConnections.pop();
     }
@@ -45,7 +48,7 @@ class NntpService {
       return this._createConnection();
     }
 
-    return new Promise(resolve => this.waiters.push(resolve));
+    return new Promise(resolve => (low ? this.lowWaiters : this.waiters).push(resolve));
   }
 
   /**
@@ -58,7 +61,7 @@ class NntpService {
       this.discard(conn);
       return;
     }
-    const resolve = this.waiters.shift();
+    const resolve = this.nextWaiter();
     if (resolve) {
       resolve(conn);
     } else {
@@ -73,13 +76,17 @@ class NntpService {
     this._destroy(conn);
 
     // Someone is waiting for a slot: open a fresh connection for them.
-    const resolve = this.waiters.shift();
+    const resolve = this.nextWaiter();
     if (resolve) {
       this.createdCount++;
       this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
     }
   }
 
+  private nextWaiter() {
+    return this.waiters.shift() ?? this.lowWaiters.shift();
+  }
+
   private _destroy(conn: any) {
     try {
       conn.sock?.destroy();
@@ -126,6 +133,7 @@ class NntpService {
     this.allConnections.clear();
     this.idleConnections.length = 0;
     this.waiters.length = 0;
+    this.lowWaiters.length = 0;
     this.createdCount = 0;
   }
 }

app/services/NzbImportService.ts

@@ -7,9 +7,9 @@ import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
 import { parseSubject } from '#lib/subject/index'
 import { fileStem } from '#lib/release/index'
 import { looksRandom } from '#services/ArchiveService'
-import QueueService from '#services/QueueService'
 import FileStore from '#services/FileStore'
 import { isArchiveStart } from '#lib/content/index'
+import InspectionQueue from '#services/InspectionQueue'
 
 export interface NzbImportResult {
   releaseId: number
@@ -207,9 +207,7 @@ class NzbImportService {
     // Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
     const archives = added.filter((f) => isArchiveStart(f.filename))
     if (archives.length) {
-      await QueueService.collectionQueue.addBulk(
-        archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
-      )
+      await InspectionQueue.add(archives.map((f) => Number(f.id)))
     }
     return result
   }

app/services/Par2Service.ts

@@ -1,15 +1,10 @@
 import db from '@adonisjs/lucid/services/db'
-import type { Segment } from '#lib/nzb/index'
 import logger from '@adonisjs/core/services/logger'
-import NntpService from '#services/NntpService'
 import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
-import { decodeSegment, YencFile } from '#lib/yenc/index'
-import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
+import type { Par2Set } from '#lib/par2/index'
 import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
 import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
 
-/** Par2 index files are small; recovery volumes can be large and are not needed. */
-const MAX_PARTS = 20
 /** Size-only matching below this is too ambiguous. */
 const SIZE_ONLY_MIN = 1024 * 1024
 
@@ -32,34 +27,6 @@ export interface Par2Result {
  * body-derived binaries carry their exact size. See CONCEPT.md (D16).
  */
 class Par2Service {
-  /**
-   * Reads a par2 file whose first segment is already decoded, downloading the
-   * remaining parts if there are any. Returns null when it is not par2.
-   */
-  async read(first: Buffer, segments: Segment[]): Promise<Par2Set | null> {
-    if (!isPar2(first)) return null
-    const ids = segments.map((part) => part.id)
-    if (ids.length > MAX_PARTS) return null
-    if (ids.length === 1) return parsePar2(first)
-
-    let assembled: YencFile | undefined
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      for (const id of ids) {
-        const segment = decodeSegment(await NntpService.fetchBody(conn, id))
-        assembled ??= YencFile.fromSegment(segment)
-        assembled.add(segment)
-      }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    return parsePar2(assembled!.getBuffer())
-  }
-
   async attach(binary: any, set: Par2Set): Promise<Par2Result> {
     const result = await db.transaction(async (trx) => {
       // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull

app/services/QueueService.ts

@@ -41,6 +41,10 @@ class QueueService {
   get collectionQueue() {
     return this.queue('collection-queue')
   }
+  /** Complete archive listings, at low priority (ArchiveWalkService). */
+  get archiveWalkQueue() {
+    return this.queue('archive-walk-queue')
+  }
 
   /** Queues no longer used by the pipeline, kept here so GC can empty them. */
   get legacy(): Queue[] {
@@ -48,7 +52,7 @@ class QueueService {
   }
 
   get all(): Queue[] {
-    return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue]
+    return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue, this.archiveWalkQueue]
   }
 
   /** Closes the queues that were opened. */

app/services/ReleaseCompleteness.ts

@@ -0,0 +1,35 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** Noted releases taken per transaction. */
+const BATCH = 2000
+
+/**
+ * Release completeness (D24) for releases whose files or par2 listing
+ * changed: triggers only note them in release_dirty (a release can be noted
+ * thousands of times between sweeps), and gc:scheduler recomputes each
+ * noted release once every RELEASE_COMPLETE_INTERVAL_MS (LOG.md 2026-10-06 (6)).
+ */
+class ReleaseCompleteness {
+  /** Recomputes all noted releases; returns how many distinct releases were checked. */
+  async sweep(): Promise<number> {
+    let checked = 0
+    for (;;) {
+      const done = await db.transaction(async (trx) => {
+        // Taken and recomputed in one transaction: a failure leaves the notes for the next sweep.
+        const { rows } = await trx.rawQuery(
+          `DELETE FROM release_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM release_dirty LIMIT ?)) RETURNING release_id`,
+          [BATCH]
+        )
+        const ids: number[] = [...new Set<number>(rows.map((row: { release_id: string }) => Number(row.release_id)))]
+        if (ids.length > 0) {
+          await trx.rawQuery(`SELECT recompute_release_complete(id) FROM unnest(?::bigint[]) AS id`, [ids])
+        }
+        checked += ids.length
+        return rows.length < BATCH
+      })
+      if (done) return checked
+    }
+  }
+}
+
+export default new ReleaseCompleteness()

app/services/RemoteContentService.ts

@@ -0,0 +1,168 @@
+import { randomUUID } from 'node:crypto'
+import { Job, Worker } from 'bullmq'
+import logger from '@adonisjs/core/services/logger'
+import queueConfig from '#config/queue'
+import nntpConfig from '#config/nntp'
+import QueueService from '#services/QueueService'
+import NntpService from '#services/NntpService'
+import ContentService, { describeContent } from '#services/ContentService'
+import ArchiveWalkService from '#services/ArchiveWalkService'
+import { isMissingArticle, type ContentTask, type InspectProbe, type InspectTask, type WalkProbe } from '#lib/content/probe'
+
+/** Inspection first; archive walks (D40) only when it has nothing waiting. */
+const QUEUES = ['collection-queue', 'archive-walk-queue'] as const
+type QueueName = (typeof QUEUES)[number]
+
+/** A node has this long to post a job's result; after that the job goes back to the queue (BullMQ's stalled check). */
+const LOCK_MS = 10 * 60 * 1000
+
+export interface RemoteJob {
+  queue: QueueName
+  jobId: string
+  /** The job's lock: whoever holds it may complete the job. */
+  token: string
+  task: ContentTask
+}
+
+export class JobLost extends Error {}
+export class ArticleRelayOff extends Error {}
+
+/**
+ * Content inspection and archive walks on seeding nodes (CONCEPT.md D41).
+ * Nodes take jobs from the same queues worker:collection works on: a lease
+ * locks a job (BullMQ's manual processing) and gives the node a task that
+ * was prepared from the database; the node probes it, fetching articles
+ * through the relay (articles()), and posts the result, which is applied
+ * here. A job whose node goes quiet is unlocked after LOCK_MS and picked up
+ * again by whoever comes next.
+ */
+class RemoteContentService {
+  private readonly workers = new Map<QueueName, Worker>()
+
+  /** Up to `max` jobs for a node, each locked for LOCK_MS. */
+  async lease(nodeId: number, max: number): Promise<RemoteJob[]> {
+    const jobs: RemoteJob[] = []
+    for (const queue of QUEUES) {
+      while (jobs.length < max) {
+        const token = `node-${nodeId}:${randomUUID()}`
+        const job = await this.worker(queue).getNextJob(token, { block: false })
+        if (!job) break
+        try {
+          const task = await this.prepare(queue, job)
+          if (!task) {
+            await job.moveToCompleted(null, token, false)
+            continue
+          }
+          // What applying needs, kept with the job: the node's copy of the task isn't trusted.
+          if (task.kind === 'inspect') await job.updateData({ ...job.data, remote: { fileId: task.fileId, name: task.name } })
+          jobs.push({ queue, jobId: job.id!, token, task })
+        } catch (error: any) {
+          await job.moveToFailed(error, token, false).catch(() => {})
+        }
+      }
+    }
+    return jobs
+  }
+
+  /** Applies a node's result and completes the job. */
+  async finish(queue: QueueName, jobId: string, token: string, result: InspectProbe | WalkProbe) {
+    const job = await this.job(queue, jobId)
+    let summary: unknown
+    if (queue === 'collection-queue') {
+      const remote = job.data.remote
+      if (!remote) throw new JobLost(`Job ${jobId} was not leased to a node`)
+      const task: InspectTask = { kind: 'inspect', fileId: Number(remote.fileId), name: String(remote.name), segments: [], sevenZip: null }
+      const content = await ContentService.apply(task, result as InspectProbe)
+      for (const line of describeContent(task.name, content)) logger.info(`[node] ${line}`)
+      summary = content.contentType
+    } else {
+      const outcome = await ArchiveWalkService.apply(Number(job.data.releaseId), result as WalkProbe)
+      logger.info(
+        `[node] Archive walk of release ${outcome.releaseId}: ${outcome.before} → ${outcome.entries} entries from ${outcome.segments} segments` +
+          (outcome.complete ? '' : ` (stopped: ${outcome.stopped})`) +
+          (outcome.renamed ? `, renamed to "${outcome.renamed}"` : '')
+      )
+      summary = outcome
+    }
+    await this.locked(() => job.moveToCompleted(summary, token, false))
+  }
+
+  /** The probe failed on the node (e.g. a missing article): retried like a local failure. */
+  async fail(queue: QueueName, jobId: string, token: string, error: string) {
+    const job = await this.job(queue, jobId)
+    // A missing first article from a node that predates the probe's own handling: a result,
+    // not something to retry (each try costs Eweka ~450 ms; LOG.md 2026-10-06 (14)).
+    if (queue === 'collection-queue' && job.data.remote && isMissingArticle(error)) {
+      return this.finish(queue, jobId, token, { size: 0, contentType: 'missing', md5_16k: null, yencName: '' })
+    }
+    await this.locked(() => job.moveToFailed(new Error(`node: ${error}`), token, false))
+  }
+
+  /** A stopping node gives its jobs back, without using up an attempt. */
+  async release(queue: QueueName, jobId: string, token: string) {
+    const job = await this.job(queue, jobId)
+    await this.locked(() => job.moveToWait(token))
+  }
+
+  /** One article body for a relay node, on this process's NNTP pool. Null when the server doesn't have it. */
+  async article(messageId: string): Promise<Buffer | null> {
+    // NNTP_POOL_SIZE 0 (SEED_ARTICLE_CONNECTIONS=0): no connections for this; acquire() would wait forever.
+    if (nntpConfig.poolSize <= 0) throw new ArticleRelayOff('Article relay is off (SEED_ARTICLE_CONNECTIONS=0)')
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
+    } catch (error: any) {
+      // 430 no such article: a fact about the article, the connection is fine.
+      if (/^4\d\d/.test(String(error.message))) return null
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+
+  /** Closes the lease workers' Redis connections (tests; a server just exits). */
+  async close() {
+    await Promise.all([...this.workers.values()].map((worker) => worker.close()))
+    this.workers.clear()
+  }
+
+  isQueue(name: unknown): name is QueueName {
+    return QUEUES.includes(name as QueueName)
+  }
+
+  private async prepare(queue: QueueName, job: Job): Promise<ContentTask | null> {
+    if (queue === 'collection-queue') return ContentService.prepare(Number(job.data.fileId))
+    const prepared = await ArchiveWalkService.prepare(Number(job.data.releaseId))
+    return 'task' in prepared ? prepared.task : null
+  }
+
+  private async job(queue: QueueName, jobId: string): Promise<Job> {
+    const job = await Job.fromId(queue === 'collection-queue' ? QueueService.collectionQueue : QueueService.archiveWalkQueue, jobId)
+    if (!job) throw new JobLost(`Job ${jobId} is gone`)
+    return job
+  }
+
+  /** BullMQ refuses a job whose lock another holder has now: the node's result came too late. */
+  private async locked(action: () => Promise<unknown>) {
+    try {
+      await action()
+    } catch (error: any) {
+      if (/lock|not in the active state|Missing key/i.test(String(error.message))) throw new JobLost(error.message)
+      throw error
+    }
+  }
+
+  /** A worker that only hands out jobs (no processor, never runs). */
+  private worker(queue: QueueName): Worker {
+    let worker = this.workers.get(queue)
+    if (!worker) {
+      worker = new Worker(queue, null, { connection: queueConfig.connection, lockDuration: LOCK_MS, autorun: false })
+      this.workers.set(queue, worker)
+    }
+    return worker
+  }
+}
+
+export default new RemoteContentService()

app/services/SearchService.ts

@@ -70,8 +70,8 @@ const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
 
 /** Incomplete, and every file is a par2 file: nothing to download but repair data. */
 const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
-  SELECT 1 FROM files f WHERE f.release_id = r.id
-    AND f.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
+  SELECT 1 FROM files f LEFT JOIN file_details d ON d.file_id = f.id WHERE f.release_id = r.id
+    AND d.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
 
 /** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
 function pattern(word: string): string {

app/services/SeedFetchPool.ts

@@ -8,8 +8,12 @@ interface Slot {
   usedAt: number
 }
 
-/** A pooled connection unused this long is replaced: the news server may have dropped it. */
-const IDLE_MS = 2 * 60 * 1000
+/**
+ * A pooled connection unused this long is replaced: the news server drops
+ * idle connections without telling, and a request on one then hangs until
+ * the inactivity timeout (LOG.md 2026-10-05 (11)).
+ */
+const IDLE_MS = 60 * 1000
 
 /**
  * NNTP connections in the seed API for relay nodes (CONCEPT.md D38): the
@@ -26,7 +30,46 @@ class SeedFetchPool {
     return indexerConfig.seed.nntpConnections
   }
 
+  /**
+   * Pooled connections fetch compressed unless SEED_NNTP_COMPRESS=false. A failure is retried once without compression, on a connection of its
+   * own: either the pooled connection was dead (dropped while idle), or the
+   * server stalled mid-reply in compressed mode. Eweka does that for some
+   * chunks: it stops after most of the stream and never sends the rest
+   * (a.b.ath 49701720000: 19,672 of 20,000 overviews, then silence), while
+   * the same chunk uncompressed comes in 0.3 s (LOG.md 2026-10-05 (11)).
+   */
   async fetch(group: string, from: number, to: number): Promise<Buffer> {
+    try {
+      return await this.fetchOnce(group, from, to)
+    } catch {
+      return this.fetchPlain(group, from, to)
+    }
+  }
+
+  private async fetchPlain(group: string, from: number, to: number): Promise<Buffer> {
+    const client = await this.connect(false)
+    try {
+      await client.group(group)
+      return await client.xoverRaw(from, to)
+    } finally {
+      client.close()
+    }
+  }
+
+  private connect(compress: boolean) {
+    return XoverClient.connect({
+      host: nntpConfig.host,
+      port: nntpConfig.port,
+      secure: nntpConfig.secure,
+      servername: nntpConfig.tlsServername,
+      user: nntpConfig.user,
+      password: nntpConfig.password?.release(),
+      timeoutMs: indexerConfig.seed.nntpTimeoutMs,
+      compress,
+    })
+  }
+
+  private async fetchOnce(group: string, from: number, to: number): Promise<Buffer> {
     const slot = await this.acquire()
     let ok = false
     try {
@@ -42,6 +85,14 @@ class SeedFetchPool {
     }
   }
 
+  /** Closes the idle connections (tests, shutdown); busy ones close when released. */
+  closeIdle() {
+    for (const slot of this.idle.splice(0)) {
+      slot.client.close()
+      this.open--
+    }
+  }
+
   private async acquire(): Promise<Slot> {
     while (true) {
       const slot = this.idle.pop()
@@ -54,17 +105,7 @@ class SeedFetchPool {
       if (this.open < this.size) {
         this.open++
         try {
-          const client = await XoverClient.connect({
-            host: nntpConfig.host,
-            port: nntpConfig.port,
-            secure: nntpConfig.secure,
-            servername: nntpConfig.tlsServername,
-            user: nntpConfig.user,
-            password: nntpConfig.password?.release(),
-            timeoutMs: nntpConfig.commandTimeoutMs,
-            compress: true,
-          })
-          return { client, usedAt: Date.now() }
+          return { client: await this.connect(indexerConfig.seed.nntpCompress), usedAt: Date.now() }
         } catch (error) {
           this.open--
           this.waiters.shift()?.()

app/services/SeedOrchestrator.ts

@@ -1,5 +1,6 @@
 import { createHash, randomBytes } from 'node:crypto'
 import db from '@adonisjs/lucid/services/db'
+import { cut, uncovered } from '#lib/seed/plan'
 import NntpService from '#services/NntpService'
 import SeedService from '#services/SeedService'
 import BackfillService from '#services/BackfillService'
@@ -60,18 +61,33 @@ class SeedOrchestrator {
    */
   async lease(nodeId: number): Promise<SeedLease | null> {
     if ((await BackfillService.room()) !== true) return null
+    // Fully reported but never finished (the node went away at the very end): nothing left to fetch.
+    await db.rawQuery(
+      `UPDATE seed_ranges SET status = 'done', done_at = now(), node_id = NULL
+       WHERE next_id > end_id AND (status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))`,
+      [LEASE_TTL_SECONDS]
+    )
     // Ranges that broke node after node stop being handed out.
     await db.rawQuery(
       `UPDATE seed_ranges SET status = 'failed', error = coalesce(error, 'lease expired too often')
        WHERE status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?) AND attempts >= ?`,
       [LEASE_TTL_SECONDS, MAX_ATTEMPTS]
     )
+    // Priorities take turns: while ranges of a higher priority are leased for less than half of
+    // the work in progress they go first, otherwise the rest does (a 2026 gap fetches ~5x slower
+    // than older history; LOG.md 2026-10-06 (10)).
+    const { rows: mix } = await db.rawQuery(
+      `SELECT count(*) FILTER (WHERE priority > 0) AS high, count(*) AS all FROM seed_ranges
+       WHERE status = 'leased' AND heartbeat_at >= now() - make_interval(secs => ?)`,
+      [LEASE_TTL_SECONDS]
+    )
+    const order = Number(mix[0].high) * 2 < Number(mix[0].all) + 1 ? 'priority DESC, id' : 'priority ASC, id'
     const { rows } = await db.rawQuery(
       `UPDATE seed_ranges SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1
        WHERE id = (
          SELECT id FROM seed_ranges
          WHERE status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?))
-         ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED)
+         ORDER BY ${order} LIMIT 1 FOR UPDATE SKIP LOCKED)
        RETURNING id, group_name, next_id, end_id`,
       [nodeId, LEASE_TTL_SECONDS]
     )
@@ -83,7 +99,7 @@ class SeedOrchestrator {
   async batch(nodeId: number, rangeId: number, batch: SeedBatch) {
     const range = await this.leased(nodeId, rangeId)
     const stored = await SeedService.store(batch.files.map(normalizeSeedFile))
-    await SeedService.storeFragments(range.group_name, batch.fragments)
+    await SeedService.storeFragments(batch.fragments, { start: Number(range.start_id), end: Number(range.end_id) })
     await db.rawQuery(
       `UPDATE seed_ranges SET next_id = greatest(next_id, least(?, end_id + 1)), heartbeat_at = now(),
          headers = headers + ?, files = files + ?, fragments = fragments + ?
@@ -137,9 +153,12 @@ class SeedOrchestrator {
    * server's oldest article. A group's backfill stops where its seed plan
    * begins. Ranges are interleaved across groups, newest first.
    */
-  async plan(groups: string[], options: { days?: number; size: number }): Promise<{ group: string; from: number; to: number; ranges: number }[]> {
+  async plan(
+    groups: string[],
+    options: { days?: number; size: number; priority?: number }
+  ): Promise<{ group: string; from: number; to: number; ranges: number; gaps: number }[]> {
     const perGroup: { group: string; ranges: [number, number][] }[] = []
-    const summary: { group: string; from: number; to: number; ranges: number }[] = []
+    const summary: { group: string; from: number; to: number; ranges: number; gaps: number }[] = []
     const conn = await NntpService.acquire()
     let failed = false
     try {
@@ -153,20 +172,21 @@ class SeedOrchestrator {
           const date = header ? Date.parse(header.date) : Number.NaN
           return Number.isNaN(date) ? null : date
         }
-        // Top: where live indexing began, or the backfill's position, or below earlier seed ranges.
+        // Top: where live indexing began, or as far down as the backfill has come.
         let top = Number(await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf))
+        if (group.backfill_next_id !== null) top = Math.min(top, Number(group.backfill_next_id))
         const backfilling = group.backfill_next_id !== null && group.backfill_until_id !== null && Number(group.backfill_next_id) > Number(group.backfill_until_id)
-        if (backfilling) top = Math.min(top, Number(group.backfill_next_id))
-        const seeded = await db.from('seed_ranges').where('group_name', name).min('start_id as low').first()
-        if (seeded?.low !== null && seeded?.low !== undefined) top = Math.min(top, Number(seeded.low))
         const bottom = options.days
           ? Number(await articleAtDate(first, last, Date.now() - options.days * 24 * 60 * 60 * 1000, dateOf))
           : Number(first)
 
-        const ranges: [number, number][] = []
-        for (let end = top - 1; end >= bottom; end -= options.size) ranges.push([Math.max(bottom, end - options.size + 1), end])
+        // Everything below the top that no earlier range covers (in any state). Planning below
+        // the lowest earlier range left a 13-month gap in a.b.ath (LOG.md 2026-10-06 (7)).
+        const planned = await db.from('seed_ranges').where('group_name', name).select('start_id', 'end_id')
+        const gaps = uncovered(bottom, top - 1, planned.map((r) => [Number(r.start_id), Number(r.end_id)] as [number, number]))
+        const ranges = cut(gaps, options.size)
         perGroup.push({ group: name, ranges })
-        summary.push({ group: name, from: bottom, to: top - 1, ranges: ranges.length })
+        summary.push({ group: name, from: bottom, to: top - 1, ranges: ranges.length, gaps: gaps.length })
         // The seed plan takes over the rest of this group's backfill.
         if (backfilling && ranges.length > 0) {
           await db.from('groups').where('id', group.id).update({ backfill_until_id: group.backfill_next_id })
@@ -184,7 +204,7 @@ class SeedOrchestrator {
     for (let i = 0; perGroup.some((g) => i < g.ranges.length); i++) {
       for (const g of perGroup) {
         const range = g.ranges[i]
-        if (range) rows.push({ group_name: g.group, start_id: range[0], end_id: range[1], next_id: range[0] })
+        if (range) rows.push({ group_name: g.group, start_id: range[0], end_id: range[1], next_id: range[0], priority: options.priority ?? 0 })
       }
     }
     for (let i = 0; i < rows.length; i += 5000) await db.table('seed_ranges').multiInsert(rows.slice(i, i + 5000))

app/services/SeedService.ts

@@ -1,11 +1,14 @@
 import db from '@adonisjs/lucid/services/db'
 import IngestService from '#services/IngestService'
-import FileStore from '#services/FileStore'
+import FileStore, { insertLimits } from '#services/FileStore'
 import SfvService from '#services/SfvService'
 import ContentService from '#services/ContentService'
-import QueueService from '#services/QueueService'
-import StagingService from '#services/StagingService'
 import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
+import InspectionQueue from '#services/InspectionQueue'
+import GarbageFilter from '#services/GarbageFilter'
+
+/** Articles from a range's edge within which a file may continue in the neighbouring range. */
+const EDGE_ARTICLES = 50_000
 
 export interface SeedStoreResult {
   files: number
@@ -13,6 +16,8 @@ export interface SeedStoreResult {
   existing: number
   releases: number
   inspections: number
+  /** Random-named files of garbage signatures or dropped releases, not stored (D42). */
+  dropped: number
 }
 
 /**
@@ -24,9 +29,17 @@ export interface SeedStoreResult {
  */
 class SeedService {
   async store(batch: SeedFile[]): Promise<SeedStoreResult> {
-    const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0 }
+    const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0, dropped: 0 }
     // A repost can complete the same file twice in one batch; ON CONFLICT DO UPDATE takes each row once.
-    const files = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
+    const unique = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
+    // Garbage isn't stored at all: no release, file or binary (CONCEPT.md D42).
+    const { kept: files, dropped } = await GarbageFilter.keep(unique, (file) => ({
+      name: file.name,
+      messageId: file.segments[0]?.[1],
+      groups: file.groups,
+      releaseKey: file.release?.key ?? file.fileKey,
+    }))
+    result.dropped = dropped
     if (files.length === 0) return result
     const knex = db.connection().getWriteClient()
     const now = new Date()
@@ -56,54 +69,83 @@ class SeedService {
     result.files = inserted.length
     result.existing = rows.length - inserted.length
 
+    // File ids for the binaries below (files queued for inspection only): new ones from the insert,
+    // existing ones looked up. Looking up every key of the batch probed all files partitions again,
+    // 15% of Postgres' active time (LOG.md 2026-10-06 (12)).
     const fileIds = new Map<string, number>()
-    for (const row of await db.from('files').whereIn('file_key', rows.map((r) => r.file_key)).select('id', 'file_key')) {
-      fileIds.set(row.file_key, Number(row.id))
+    for (const row of inserted) if (row.file_key) fileIds.set(row.file_key, row.id)
+    const missing = files.filter((file) => !fileIds.has(file.fileKey) && ContentService.isCandidate(file.name)).map((file) => file.fileKey)
+    for (let i = 0; i < missing.length; i += 5000) {
+      for (const row of await db.from('files').whereIn('file_key', missing.slice(i, i + 5000)).select('id', 'file_key')) {
+        fileIds.set(row.file_key, Number(row.id))
+      }
     }
-    // The completed binary worker:file would have left behind; an existing one just learns its file.
-    const binaries = files.map((file, i) => ({
-      file_key: file.fileKey,
-      poster: file.poster,
-      subject: file.subject,
-      name: file.name,
-      total_parts: file.totalParts,
-      groups: rows[i].groups,
-      posted_at: postedOf(file) ?? null,
-      first_seen_at: now,
-      last_seen_at: now,
-      completed_at: now,
-      release_id: rows[i].release_id,
-      file_id: fileIds.get(file.fileKey) ?? null,
-    }))
+    // A completed binary, as worker:file would have left behind, only for files queued for content
+    // inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others
+    // (~85% in a.b.ath) it was only GC churn: 1.6M rows an hour inserted and deleted (LOG.md 2026-10-06 (11)).
+    const insertedKeys = new Set(inserted.map((row) => row.file_key))
+    const binaries = files.flatMap((file, i) => {
+      if (!ContentService.isCandidate(file.name)) return []
+      return [{
+        file_key: file.fileKey,
+        poster: file.poster,
+        subject: file.subject,
+        name: file.name,
+        total_parts: file.totalParts,
+        groups: rows[i].groups,
+        posted_at: postedOf(file) ?? null,
+        first_seen_at: now,
+        last_seen_at: now,
+        completed_at: now,
+        release_id: rows[i].release_id,
+        file_id: fileIds.get(file.fileKey) ?? null,
+        // New files are queued for inspection below; an existing file was queued when it was added.
+        awaiting_inspection: insertedKeys.has(file.fileKey) ? true : null,
+      }]
+    })
     binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
-    await knex('binaries')
-      .insert(binaries)
-      .onConflict('file_key')
-      .merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
+    for (let i = 0; i < binaries.length; i += insertLimits.rows) {
+      await knex('binaries')
+        .insert(binaries.slice(i, i + insertLimits.rows))
+        .onConflict('file_key')
+        .merge({
+          file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any,
+          awaiting_inspection: knex.raw('coalesce(binaries.awaiting_inspection, excluded.awaiting_inspection)') as any,
+        })
+    }
 
     // As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
-    const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row]))
+    const rowByKey = new Map(rows.map((row) => [row.file_key, row]))
     const added = inserted.flatMap((file) => {
-      const row = rowById.get(file.id)
+      const row = file.file_key ? rowByKey.get(file.file_key) : undefined
       return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
     })
     await SfvService.claimAll(added)
     const candidates = added
       .filter((file) => ContentService.isCandidate(file.name))
-      .map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
-    if (candidates.length > 0) await QueueService.collectionQueue.addBulk(candidates)
+      .map((file) => file.id)
+    if (candidates.length > 0) await InspectionQueue.add(candidates)
     result.inspections = candidates.length
     return result
   }
 
-  /** Segments of files that didn't complete on a node: staging (D35) for lone ones, binaries and parts for the rest. */
-  async storeFragments(group: string, fragments: SeedFragment[]): Promise<void> {
-    if (fragments.length === 0) return
-    const segments = fragments.map(fromFragment)
-    const { ingest } = StagingService.enabled(group)
-      ? await StagingService.filter(segments, { counters: StagingService.countersEnabled(group) })
-      : { ingest: segments }
-    await IngestService.ingest(ingest)
+  /**
+   * Segments of files that didn't complete on a node. Only files near the
+   * range's edges can continue in the neighbouring range (worked on by some
+   * node, maybe hours later): those go straight into binaries and parts
+   * (staging would drop them after 30 minutes). The rest are files with
+   * articles missing for good, and are dropped: staging them filled 22 GB
+   * of churn on the NVMe (LOG.md 2026-10-05 (10)). Returns the segments kept.
+   */
+  async storeFragments(fragments: SeedFragment[], range: { start: number; end: number }): Promise<number> {
+    const nearEdge = new Set<string>()
+    for (const fragment of fragments) {
+      const n = Number(fragment.articleNumber)
+      if (Number.isFinite(n) && (n < range.start + EDGE_ARTICLES || n > range.end - EDGE_ARTICLES)) nearEdge.add(fragment.fileKey)
+    }
+    const kept = fragments.filter((fragment) => nearEdge.has(fragment.fileKey))
+    if (kept.length > 0) await IngestService.ingest(kept.map(fromFragment))
+    return kept.length
   }
 }
 

app/services/SevenZipService.ts

@@ -1,34 +1,10 @@
-import { execFile } from 'node:child_process'
-import { mkdtemp, open, rm } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import path from 'node:path'
-import { promisify } from 'node:util'
 import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
-import NntpService from '#services/NntpService'
-import { segmentsOf } from '#services/FileStore'
-import { decodeSegment, type DecodedSegment } from '#lib/yenc/index'
-import type { RarInfo } from '#lib/rar/index'
-import {
-  checkHeader,
-  listingStart,
-  parseBsdtarListing,
-  parseSevenZipStart,
-  sevenZipVolume,
-  type SevenZipStart,
-} from '#lib/sevenzip/index'
-
-const run = promisify(execFile)
-
-/** The tail read for a listing: header plus packed header; more is not a header. */
-const MAX_TAIL_BYTES = 16 * 1024 * 1024
+import { sevenZipVolume } from '#lib/sevenzip/index'
 
 /**
- * Lists 7z archives, also split ones (.7z.001, .002, …), from the first
- * segment of the first volume and the last segment(s) of the last one
- * (CONCEPT.md D19). Those bytes are put at their place in a sparse file the
- * size of the whole archive, which bsdtar (libarchive) lists: it only reads
- * the start and the header at the end.
+ * The database side of 7z listings (CONCEPT.md D19): which volumes make up
+ * an archive. Listing itself (first segment, tail, bsdtar) is
+ * lib/sevenzip/list.ts, run by inspection here or on a node.
  */
 class SevenZipService {
   /**
@@ -55,148 +31,6 @@ class SevenZipService {
     const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
     return max > 0 && ordered.every(Boolean) ? ordered : null
   }
-
-  /**
-   * Lists the archive whose first volume starts with `firstData` (a decoded
-   * first segment). Null when it isn't a 7z start, when volumes are missing,
-   * or when the tail doesn't hold the header its start points to.
-   */
-  async list(firstFile: any, firstData: Buffer): Promise<RarInfo | null> {
-    const start = parseSevenZipStart(firstData)
-    if (!start) return null
-    const volumes = await this.volumes(firstFile)
-    if (!volumes) return null
-    if (start.headerSize === 0) return this.info([], false, false, volumes.length > 1)
-
-    const tail = await this.readTail(volumes[volumes.length - 1], start)
-    if (!tail) return null
-
-    const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
-    try {
-      const archive = path.join(dir, 'archive.7z')
-      const handle = await open(archive, 'w')
-      try {
-        await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
-        await handle.write(tail.data, 0, tail.data.length, tail.from)
-        await handle.truncate(start.archiveSize)
-      } finally {
-        await handle.close()
-      }
-      return await this.bsdtar(archive, volumes.length > 1)
-    } finally {
-      await rm(dir, { recursive: true, force: true })
-    }
-  }
-
-  /** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
-  private async bsdtar(archive: string, split: boolean): Promise<RarInfo | null> {
-    let stdout = ''
-    try {
-      stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
-    } catch (error: any) {
-      if (/header is encrypted/i.test(String(error.stderr))) return this.info([], true, true, split)
-      logger.warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
-      return null
-    }
-    const entries = parseBsdtarListing(stdout)
-    const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
-    let encrypted = false
-    if (sample) {
-      try {
-        await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
-      } catch (error: any) {
-        // Without the data (it's all zeros here), anything else fails too: only this message counts.
-        encrypted = /content is encrypted/i.test(String(error.stderr))
-      }
-    }
-    return this.info(entries, false, encrypted, split)
-  }
-
-  private info(entries: { name: string; size: number; directory: boolean }[], headersEncrypted: boolean, encrypted: boolean, split: boolean): RarInfo {
-    return {
-      format: '7z',
-      volume: split,
-      firstVolume: true,
-      solid: false,
-      headersEncrypted,
-      truncated: false,
-      entries: entries.map((e) => ({
-        name: e.name,
-        size: e.size,
-        packedSize: 0,
-        directory: e.directory,
-        encrypted: encrypted && !e.directory,
-        splitBefore: false,
-        splitAfter: false,
-        stored: false,
-        dataOffset: 0,
-      })),
-    }
-  }
-
-  /**
-   * The archive's tail from the last volume: the header (checked against its
-   * CRC) and, for an encoded header, the packed stream before it. `from` is
-   * its position in the whole archive.
-   */
-  private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | null> {
-    const ids = new Map(segmentsOf(last).map((s) => [s.number, s]))
-    const numbers = [...ids.keys()]
-    if (numbers.length === 0) return null
-    const segments = new Map<number, DecodedSegment>()
-    const segment = async (n: number) => {
-      if (!segments.has(n)) {
-        const id = ids.get(n)?.id
-        if (!id) throw new Error(`part ${n} of ${last.filename} is missing`)
-        segments.set(n, decodeSegment(await this.body(id), { lenientSize: true }))
-      }
-      return segments.get(n)!
-    }
-
-    const lastNumber = numbers[numbers.length - 1]
-    const lastSegment = await segment(lastNumber)
-    const volumeSize = lastSegment.meta.begin.size
-    const volumeStart = start.archiveSize - volumeSize
-    const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
-    const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
-    if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
-
-    // Bytes [from, to) of the whole archive, from the last volume's parts.
-    const read = async (from: number, to: number): Promise<Buffer | null> => {
-      if (from < volumeStart) return null
-      const a = from - volumeStart
-      const b = to - volumeStart
-      const pieces: Buffer[] = []
-      for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
-        const s = await segment(n)
-        const offset = (s.meta.part?.begin ?? 1) - 1
-        if (offset !== (n - 1) * partSize) return null
-        pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
-      }
-      const data = Buffer.concat(pieces)
-      return data.length === b - a ? data : null
-    }
-
-    const header = await read(start.headerStart, start.archiveSize)
-    if (!header || !checkHeader(start, header)) return null
-    const from = Math.min(listingStart(start, header), start.headerStart)
-    if (start.archiveSize - from > MAX_TAIL_BYTES) return null
-    const data = await read(from, start.archiveSize)
-    return data ? { from, data } : null
-  }
-
-  private async body(messageId: string): Promise<Buffer> {
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
-    } catch (error: any) {
-      failed = !/^4\d\d/.test(String(error.message))
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-  }
 }
 
 export default new SevenZipService()

app/services/SfvService.ts

@@ -1,5 +1,6 @@
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
+import FileStore from '#services/FileStore'
 import { parseSfv, sfvReleaseName } from '#lib/content/index'
 
 /** Releases made by name rules alone; an SFV's listing outranks them. */
@@ -30,7 +31,7 @@ class SfvService {
       .insert(entries.map((e) => ({ release_id: releaseId, name: e.name, crc: e.crc, poster: sfvFile.poster, posted_at: postedAt })))
       .onConflict(['release_id', 'name'])
       .ignore()
-    await db.from('files').where('id', sfvFile.id).update({ content_type: 'sfv' })
+    await FileStore.setDetails(Number(sfvFile.id), { content_type: 'sfv' })
 
     const candidates = await db
       .from('files as f')

app/services/SpotService.ts

@@ -31,6 +31,9 @@ const MAX_IMAGE_SEGMENTS = 10
 /** A claimed spot whose worker died is taken again after this long. */
 const RECLAIM_AFTER = '1 hour'
 
+/** The NZB's articles were read, but they don't decode to an NZB (e.g. "invalid code lengths set"). */
+export class NzbUnreadable extends Error {}
+
 /**
  * A spot's title: from its XML, which is decoded properly (the Subject of a
  * raw header is read as ISO-8859-1), else from the header.
@@ -297,6 +300,97 @@ class SpotService {
     return read
   }
 
+  /**
+   * Takes up to `limit` listed spots whose article and NZB haven't been read,
+   * newest first, marking them so another reader skips them.
+   */
+  async claimListings(limit: number): Promise<any[]> {
+    const { rows } = await db.rawQuery(
+      `UPDATE spots SET nzb_read_at = now()
+       WHERE message_id IN (
+         SELECT message_id FROM spots WHERE status = 'listed' AND nzb_read_at IS NULL
+         ORDER BY posted_at DESC LIMIT ? FOR UPDATE SKIP LOCKED)
+       RETURNING *`,
+      [limit]
+    )
+    return rows
+  }
+
+  /**
+   * Reads a listed spot's article and NZB without importing its files
+   * (LOG.md 2026-10-08 (10)): the details (description, image), the NZB's
+   * own password (`password_source = 'nzb'`, over one from the spot's text),
+   * else one in the spot's text; a placeholder NZB hides the spot (`dummy`).
+   * Returns what it found. A connection failure gives the spot back for later.
+   */
+  async readListing(spot: any): Promise<string> {
+    try {
+      const head = await this.readHead(spot.message_id)
+      if (!head) {
+        await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
+        return 'unverified'
+      }
+      const update: Record<string, unknown> = this.details(head.xml)
+      let outcome = 'nothing'
+      let nzbXml: string | null = null
+      try {
+        nzbXml = await this.readNzb(head)
+      } catch (error: any) {
+        if (error instanceof NzbUnreadable) {
+          // Corrupt: reading it again won't help (LOG.md 2026-10-08 (12)).
+          update.error = `NZB unreadable: ${error.message}`.slice(0, 500)
+          outcome = 'nzb-unreadable'
+        } else if (/^4\d\d/.test(String(error.message))) {
+          // The NZB's articles are gone: the listing can't be downloaded any more.
+          update.error = 'NZB articles gone'
+          outcome = 'nzb-gone'
+        } else {
+          throw error
+        }
+      }
+      const nzb = nzbXml ? parseNzb(nzbXml) : null
+      if (nzb && nzb.files.length > 0 && isDummyNzb(nzb, head.header.size)) {
+        update.status = 'dummy'
+        outcome = 'dummy'
+      } else if (nzb?.meta.password && spot.release_id) {
+        await db
+          .from('releases')
+          .where('id', spot.release_id)
+          .where((q) => q.whereNull('password').orWhere('password_source', 'spot'))
+          .update({ password: nzb.meta.password, password_source: 'nzb' })
+        outcome = 'nzb-password'
+      }
+      await db.from('spots').where('message_id', spot.message_id).update(update)
+      if (outcome !== 'dummy' && (await this.applyPassword(spot.message_id)) && outcome !== 'nzb-password') outcome = 'spot-password'
+      return outcome
+    } catch (error: any) {
+      // The spot's own article is gone (430): nothing to read later either.
+      if (/^4\d\d/.test(String(error.message))) {
+        await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
+        return 'gone'
+      }
+      await db.from('spots').where('message_id', spot.message_id).update({ nzb_read_at: null })
+      logger.warn(`Spot ${spot.message_id} listing: ${error.message}`)
+      return 'retry'
+    }
+  }
+
+  /** Reads up to `limit` unread listings, `concurrency` at a time. Returns the outcomes counted. */
+  async readListings(limit: number, concurrency: number): Promise<Record<string, number>> {
+    const spots = await this.claimListings(limit)
+    const counts: Record<string, number> = {}
+    let next = 0
+    await Promise.all(
+      Array.from({ length: Math.max(1, concurrency) }, async () => {
+        while (next < spots.length) {
+          const outcome = await this.readListing(spots[next++])
+          counts[outcome] = (counts[outcome] ?? 0) + 1
+        }
+      })
+    )
+    return counts
+  }
+
   /** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
   async image(spot: any): Promise<Buffer | null> {
     const segments: string[] = spot.image?.segments ?? []
@@ -315,13 +409,23 @@ class SpotService {
     return { header, xml: parseSpotXml(decodeSpotText(xml)) }
   }
 
-  /** Downloads the spot's compressed NZB and imports it as a release (D26). */
-  private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
+  /** Downloads and decodes the spot's compressed NZB; null when it has none (or a too large one). */
+  private async readNzb(head: { xml: SpotXml | null }): Promise<string | null> {
     const segments = head.xml?.nzbSegments ?? []
-    if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' }
+    if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return null
     const bodies = []
     for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
-    const nzbXml = decodeSpotBinary(bodies).toString('utf8')
+    try {
+      return decodeSpotBinary(bodies).toString('utf8')
+    } catch (error: any) {
+      throw new NzbUnreadable(String(error.message))
+    }
+  }
+
+  /** Downloads the spot's compressed NZB and imports it as a release (D26). */
+  private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
+    const nzbXml = await this.readNzb(head)
+    if (!nzbXml) return { status: 'no-nzb' }
     const nzb = parseNzb(nzbXml)
     if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
     const { header } = head

bin/console.ts

@@ -18,6 +18,12 @@ const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
  * URL to the application root. AdonisJS need it to resolve
  * paths to file and directories for scaffolding commands
  */
+/**
+ * The command being run, for start/env.ts: Ace takes the arguments out of
+ * process.argv before the environment is loaded.
+ */
+process.env.ACE_COMMAND ??= process.argv[2] ?? ''
+
 const APP_ROOT = new URL('../', import.meta.url)
 
 /**

commands/ArchiveWalk.ts

@@ -0,0 +1,70 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import QueueService from '#services/QueueService'
+import NntpService from '#services/NntpService'
+import ArchiveWalkService from '#services/ArchiveWalkService'
+
+const BATCH = 5000
+
+/**
+ * Complete archive listings for releases inspected before archive walks
+ * existed (CONCEPT.md D40): queues those whose listing leaves most of the
+ * release unaccounted for. worker:collection walks them, one at a time and
+ * behind inspection. With release ids, walks those right here instead.
+ */
+export default class ArchiveWalk extends BaseCommand {
+  public static commandName = 'content:walk-archives'
+  public static description = 'Queues releases with partial archive listings for a (low priority) header walk.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Release ids to walk now (instead of queueing)', required: false })
+  declare releaseIds?: string[]
+
+  @flags.number({ description: 'Queue at most this many releases' })
+  declare limit?: number
+
+  @flags.boolean({ description: 'Also releases that were walked before' })
+  declare again: boolean
+
+  public async run() {
+    try {
+      if (this.releaseIds?.length) {
+        for (const id of this.releaseIds) {
+          this.logger.info(JSON.stringify(await ArchiveWalkService.walk(Number(id))))
+        }
+        return
+      }
+      let lastId = 0
+      let queued = 0
+      while (!this.limit || queued < this.limit) {
+        const query = db
+          .from('releases as r')
+          .where('r.id', '>', lastId)
+          .whereNotNull('r.contents')
+          .orderBy('r.id')
+          .limit(BATCH)
+          .select('r.id', 'r.size', 'r.contents_walked_at', db.raw(`(
+            SELECT count(*) FROM jsonb_array_elements(r.contents)) AS n`), db.raw(`(
+            SELECT coalesce(sum((e->>'size')::bigint), 0) FROM jsonb_array_elements(r.contents) e) AS listed`))
+        const rows = await query
+        if (rows.length === 0) break
+        lastId = Number(rows.at(-1).id)
+        const jobs = rows
+          .filter((r) => this.again || !r.contents_walked_at)
+          .filter((r) => ArchiveWalkService.worthWalking(Number(r.n), Number(r.listed), Number(r.size)))
+          .slice(0, this.limit ? this.limit - queued : undefined)
+          .map((r) => ({ name: 'walk', data: { releaseId: Number(r.id) }, opts: { jobId: `walk-${r.id}-${Date.now()}` } }))
+        if (jobs.length > 0) await QueueService.archiveWalkQueue.addBulk(jobs)
+        queued += jobs.length
+      }
+      this.logger.info(`Queued ${queued} releases for an archive walk.`)
+    } finally {
+      await QueueService.closeAll()
+      await NntpService.shutdown()
+    }
+  }
+}

commands/ContentBackfill.ts

@@ -4,6 +4,7 @@ import db from '@adonisjs/lucid/services/db'
 import QueueService from '#services/QueueService'
 import ContentService from '#services/ContentService'
 import { isArchiveStart } from '#lib/content/index'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
 
 const BATCH = 1000
 
@@ -47,20 +48,20 @@ export default class ContentBackfill extends BaseCommand {
     try {
       while (!this.limit || queued < this.limit) {
         // Files never inspected: no fingerprint on their binary and no archive listing.
-        const query = db.from('files as f').where('f.id', '>', lastId)
+        const query = db.from('files as f').leftJoin('file_details as d', 'd.file_id', 'f.id').where('f.id', '>', lastId)
         // Files from NZBs (spots) have no binary.
         if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
         else query.join('binaries as b', 'b.file_id', 'f.id')
         if (this.nicovideo) {
-          query.whereNull('f.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
+          query.whereNull('d.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
         } else if (this.sfv) {
-          query.whereNull('f.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
+          query.whereNull('d.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
         } else if (this.archives) {
-          query.whereNull('f.content_type').whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
+          query.whereNull('d.content_type').whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
         } else if (this.sevenZip) {
-          query.whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
-        } else if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
-        else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
+          query.whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
+        } else if (this.archiveNzbs) query.whereRaw(`d.archive::text ~* '\\.nzb"'`)
+        else if (!this.all) query.whereNull('b.md5_16k').whereNull('d.archive')
         const files = await query
           .orderBy('f.id')
           .limit(BATCH)
@@ -71,19 +72,15 @@ export default class ContentBackfill extends BaseCommand {
         const jobs = files
           .filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
           .slice(0, this.limit ? this.limit - queued : undefined)
-          .map((f) => ({
-            name: 'process-collection',
-            data: { fileId: Number(f.id) },
-            // A new id per run: BullMQ ignores an add() for an id it still knows.
-            // Targeted backfills go ahead of whatever is waiting.
-            opts: {
-              jobId: `content-${f.id}-${Date.now()}`,
-              lifo: this.archiveNzbs || this.sevenZip || this.archives || this.sfv,
-              // niconico captures are many (~190k) and only cosmetic: behind everything else.
-              ...(this.nicovideo ? { priority: 10 } : {}),
-            },
-          }))
-        if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+          .map((f) => Number(f.id))
+        // Into the inspection backlog (InspectionQueue): targeted backfills go ahead of what's waiting;
+        // niconico captures are many (~190k) and only cosmetic, so behind everything else.
+        const priority = this.nicovideo
+          ? INSPECTION_PRIORITY.low
+          : this.archiveNzbs || this.sevenZip || this.archives || this.sfv
+            ? INSPECTION_PRIORITY.targeted
+            : INSPECTION_PRIORITY.normal
+        if (jobs.length > 0) await InspectionQueue.add(jobs, priority)
         queued += jobs.length
       }
       this.logger.info(`Queued ${queued} files for content inspection.`)

commands/FilesMoveDetails.ts

@@ -0,0 +1,56 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+
+const BATCH = 50_000
+
+/**
+ * One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
+ * listings still on `files` into file_details, a partition at a time.
+ * Values written since the deploy (by the new code, into file_details) win:
+ * a file that has details already is skipped.
+ */
+export default class FilesMoveDetails extends BaseCommand {
+  public static commandName = 'files:move-details'
+  public static description = "Copies files' content types and archive listings into file_details."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const { rows: partitions } = await db.rawQuery(
+      `SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname`
+    )
+    let total = 0
+    for (const { relname } of partitions) {
+      const started = Date.now()
+      const { rows } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM ${relname}`)
+      if (rows[0].lo === null) continue
+      let copied = 0
+      // Short statements in id order: one statement over a whole partition took 30 minutes, and the
+      // garbage filter deleted files meanwhile, which failed the foreign key (LOG.md 2026-10-08 (15)).
+      for (let lo = Number(rows[0].lo); lo <= Number(rows[0].hi); lo += BATCH) {
+        for (let attempt = 1; ; attempt++) {
+          try {
+            const result = await db.rawQuery(
+              `INSERT INTO file_details (file_id, content_type, archive)
+               SELECT id, content_type, archive FROM ${relname}
+               WHERE id >= ? AND id < ? AND (content_type IS NOT NULL OR archive IS NOT NULL)
+               ON CONFLICT (file_id) DO NOTHING`,
+              [lo, lo + BATCH]
+            )
+            copied += result.rowCount ?? 0
+            break
+          } catch (error: any) {
+            // A file deleted while this batch ran: the batch again, without it.
+            if (error.code !== '23503' || attempt >= 5) throw error
+          }
+        }
+      }
+      total += copied
+      this.logger.info(`${relname}: ${copied} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
+    }
+    this.logger.info(`${total} files' details copied.`)
+  }
+}

commands/GarbageLearn.ts

@@ -0,0 +1,38 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import GarbageFilter from '#services/GarbageFilter'
+
+/**
+ * Learns the garbage upload signatures now (gc:scheduler does it every
+ * GARBAGE_LEARN_INTERVAL_MS) and lists them (CONCEPT.md D42). --seed first
+ * fills signature_probes from a sample of the files probed before it
+ * existed, so there is something to learn from right away.
+ */
+export default class GarbageLearn extends BaseCommand {
+  public static commandName = 'garbage:learn'
+  public static description = 'Learns which upload signatures are garbage and lists them.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'First add probed releases from TABLESAMPLE SYSTEM (percent) of files' })
+  declare seed?: number
+
+  public async run() {
+    if (this.seed) {
+      const started = Date.now()
+      const added = await GarbageFilter.seed(this.seed)
+      this.logger.info(`Seeded ${added} probed releases from ${this.seed}% of files in ${Math.round((Date.now() - started) / 1000)} s.`)
+    }
+    const { signatures, dropped, changed } = await GarbageFilter.learn()
+    for (const line of changed) this.logger.info(`Changed: ${line}`)
+    const rows = await db.from('upload_signatures').orderBy('probed', 'desc').limit(40)
+    for (const row of rows) {
+      const rate = row.probed ? ((100 * row.named) / row.probed).toFixed(1) : '-'
+      this.logger.info(`${row.dropped ? 'DROP' : 'keep'}  ${row.tool.padEnd(10)} ${(row.group_name || '(no group)').padEnd(36)} ${String(row.named).padStart(6)}/${String(row.probed).padEnd(6)} named (${rate}%)`)
+    }
+    this.logger.info(`${dropped} of ${signatures} signatures dropped.`)
+  }
+}

commands/GcScheduler.ts

@@ -9,6 +9,10 @@ import IndexMaintenance from '#services/IndexMaintenance'
 import ReleaseMergeService from '#services/ReleaseMergeService'
 import StagingService from '#services/StagingService'
 import FilePartitions from '#services/FilePartitions'
+import ReleaseCompleteness from '#services/ReleaseCompleteness'
+import FileKeys from '#services/FileKeys'
+import GarbageFilter from '#services/GarbageFilter'
+import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
 
 const CLEAN_BATCH = 1000
@@ -30,6 +34,13 @@ export default class GcScheduler extends BaseCommand {
   @flags.boolean({ description: 'Report what would be removed without changing anything' })
   declare dryRun: boolean
 
+  /** When release completeness was last recomputed (it runs less often than the sweep). */
+  private completenessAt = 0
+  /** When the garbage signatures were last learned (D42). */
+  private learnedAt = 0
+  /** When split uploads were last merged. */
+  private mergedAt = 0
+
   public async run() {
     this.logger.info(
       `GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
@@ -42,6 +53,21 @@ export default class GcScheduler extends BaseCommand {
         } catch (err: any) {
           this.logger.error(`GC sweep failed: ${err.message}`)
         }
+        try {
+          await this.releaseCompleteness()
+        } catch (err: any) {
+          this.logger.error(`Release completeness failed: ${err.message}`)
+        }
+        try {
+          await this.fillFileKeys()
+        } catch (err: any) {
+          this.logger.error(`file_keys fill failed: ${err.message}`)
+        }
+        try {
+          await this.learnGarbage()
+        } catch (err: any) {
+          this.logger.error(`Garbage signatures failed: ${err.message}`)
+        }
         if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
       } while (!this.once && !Shutdown.requested)
     } finally {
@@ -62,8 +88,15 @@ export default class GcScheduler extends BaseCommand {
       }
     }
 
-    // Inserts fail without a partition for their id (D39).
-    if (!this.dryRun) await FilePartitions.ensure()
+    // Inserts fail without a partition for their id (D39). Four are kept ahead, so a failure here
+    // can wait for the next sweep; it used to end the sweep, deletes and all (LOG.md 2026-10-07 (4)).
+    if (!this.dryRun) {
+      try {
+        await FilePartitions.ensure()
+      } catch (err: any) {
+        this.logger.error(`files partitions: ${err.message}; trying again next sweep.`)
+      }
+    }
 
     const trimmed = await this.trimBodyQueue()
     if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`)
@@ -100,14 +133,46 @@ export default class GcScheduler extends BaseCommand {
       )
     }
 
-    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep.
-    const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
-    if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
+    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep. It reads
+    // every release and its files: hourly, not every sweep (LOG.md 2026-10-06 (12)).
+    if (this.once || Date.now() - this.mergedAt >= gcConfig.mergeIntervalMs) {
+      this.mergedAt = Date.now()
+      const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+      if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
+    }
 
     await this.rebuildIndexes()
   }
 
   /** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
+  /** The keys of files from before file_keys, a slice of time per sweep, until done. */
+  private async fillFileKeys() {
+    if (this.dryRun || (await FileKeys.complete())) return
+    const started = Date.now()
+    const { read, complete, nextId, untilId } = await FileKeys.fill(gcConfig.fileKeysFillMs)
+    this.logger.info(
+      `file_keys: ${read.toLocaleString('en')} keys copied in ${Math.round((Date.now() - started) / 1000)} s, ` +
+        (complete ? 'complete: file inserts use the filter now.' : `at file ${nextId.toLocaleString('en')} of ${untilId.toLocaleString('en')}.`)
+    )
+  }
+
+  private async releaseCompleteness() {
+    if (this.dryRun || (!this.once && Date.now() - this.completenessAt < gcConfig.releaseCompleteIntervalMs)) return
+    this.completenessAt = Date.now()
+    const started = Date.now()
+    const checked = await ReleaseCompleteness.sweep()
+    if (checked > 0) this.logger.info(`Release completeness: ${checked} changed releases checked in ${Math.round((Date.now() - started) / 1000)} s.`)
+  }
+
+  /** Learns which upload signatures are garbage from the probed releases (CONCEPT.md D42). */
+  private async learnGarbage() {
+    if (this.dryRun || (!this.once && Date.now() - this.learnedAt < indexerConfig.garbage.learnIntervalMs)) return
+    this.learnedAt = Date.now()
+    const { signatures, dropped, changed } = await GarbageFilter.learn()
+    this.logger.info(`Garbage signatures: ${dropped} of ${signatures} dropped.`)
+    for (const line of changed) this.logger.info(`Garbage signature ${line}`)
+  }
+
   private async rebuildIndexes() {
     const key = 'gc:reindex:last'
     const last = Number((await RedisService.client.get(key)) ?? 0)
@@ -144,19 +209,27 @@ export default class GcScheduler extends BaseCommand {
    * already live on in `files`; conflicted ones will never become a file.
    * Completed ones still waiting for worker:file are kept.
    */
+  /**
+   * Binaries without new segments for staleAfterSeconds: incomplete ones are given up on,
+   * finished ones have their file. A finished one whose file waits for content inspection
+   * stays until it's inspected (its fingerprint names it from par2 sets, D16), at most
+   * awaitInspectionSeconds (LOG.md 2026-10-06 (11)). Three deletes, one per partial index,
+   * so none reads past millions of waiting rows.
+   */
   private async deleteStaleBinaries(): Promise<number> {
-    const query = db
-      .from('binaries')
-      .where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
-      .where((q) =>
-        q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
-      )
-
-    if (this.dryRun) {
-      const [{ count }] = await query.count('* as count')
-      return Number(count)
+    const stale = new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000)
+    const waited = new Date(Date.now() - gcConfig.binaries.awaitInspectionSeconds * 1000)
+    const deletes = [
+      db.from('binaries').where('last_seen_at', '<', stale).whereNotNull('file_id').whereRaw('awaiting_inspection IS NOT TRUE'),
+      db.from('binaries').where('last_seen_at', '<', stale).where((q) => q.whereNull('completed_at').orWhereNotNull('conflicted_at')),
+      db.from('binaries').where('last_seen_at', '<', waited).whereRaw('awaiting_inspection IS TRUE'),
+    ]
+    let total = 0
+    for (const query of deletes) {
+      if (this.dryRun) total += Number((await query.count('* as count'))[0].count)
+      else total += Number(await query.delete())
     }
-    return Number(await query.delete())
+    return total
   }
 
   private async deleteOldDeferredFiles(): Promise<number> {
@@ -249,6 +322,7 @@ export default class GcScheduler extends BaseCommand {
            SELECT id FROM binaries
            WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
              AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
+             AND awaiting_inspection IS NOT TRUE
            ORDER BY last_seen_at LIMIT ?)`,
         [cutoff, gcConfig.deleteBatch]
       )

commands/IndexScheduler.ts

@@ -5,6 +5,7 @@ import NntpService from '#services/NntpService'
 import QueueService from '#services/QueueService'
 import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
+import InspectionQueue from '#services/InspectionQueue'
 import FetchRangeService from '#services/FetchRangeService'
 import SplitService from '#services/SplitService'
 import BackfillService from '#services/BackfillService'
@@ -108,6 +109,12 @@ export default class IndexScheduler extends BaseCommand {
       }
     }
 
+    // The inspection backlog lives in Postgres; BullMQ gets a working set (LOG.md 2026-10-07 (8)).
+    const feedInspection = async () => {
+      const moved = await InspectionQueue.feed()
+      if (moved > 0) this.logger.info(`Inspection: ${moved} files moved from the backlog to the queue.`)
+    }
+
     const backfill = async () => {
       try {
         const queued = await BackfillService.tick()
@@ -117,11 +124,18 @@ export default class IndexScheduler extends BaseCommand {
       }
     }
 
-    // Run immediately, then wait a full interval after each run so runs never overlap.
+    // Run immediately, then wait a full interval after each run so runs never overlap. A failed
+    // round (e.g. Postgres restarting) is logged and the next one tries again: an error that escaped
+    // here once ended the loop while the process stayed up, and live indexing stopped for ~3 hours
+    // (LOG.md 2026-10-07 (3)).
     while (!Shutdown.requested) {
-      await schedule()
-      await backfill()
-      await split()
+      for (const step of [schedule, backfill, split, feedInspection]) {
+        try {
+          await step()
+        } catch (err: any) {
+          this.logger.error(`Scheduler round failed, retrying next round: ${err.message}`)
+        }
+      }
       await Shutdown.sleep(indexerConfig.intervalMs)
     }
     await this.pool.shutdown()

commands/InspectionFromRedis.ts

@@ -0,0 +1,56 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import QueueService from '#services/QueueService'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
+
+/**
+ * One-off (LOG.md 2026-10-07 (8)): moves the inspection jobs waiting in
+ * BullMQ into the inspection_queue table, then empties BullMQ's waiting and
+ * prioritized jobs so Redis gives the memory back. The file id is read from
+ * the job id (file-<id>, content-<id>-<time>), so no job is loaded.
+ * index-scheduler then feeds a working set back in.
+ */
+export default class InspectionFromRedis extends BaseCommand {
+  public static commandName = 'inspection:from-redis'
+  public static description = 'Moves waiting inspection jobs from Redis into the inspection_queue table.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count, change nothing' })
+  declare dryRun: boolean
+
+  public async run() {
+    const queue = QueueService.collectionQueue
+    const client = await queue.client
+    const prefix = `${queue.opts.prefix ?? 'bull'}:${queue.name}`
+    try {
+      const idOf = (jobId: string) => {
+        const m = /^(?:file|content)-(\d+)/.exec(jobId)
+        return m ? Number(m[1]) : null
+      }
+      let moved = 0
+      let skipped = 0
+      const take = async (ids: string[], priority: number) => {
+        const fileIds = ids.map(idOf).filter((id): id is number => id !== null)
+        skipped += ids.length - fileIds.length
+        if (!this.dryRun) await InspectionQueue.add(fileIds, priority)
+        moved += fileIds.length
+      }
+      const waiting = await client.llen(`${prefix}:wait`)
+      for (let i = 0; i < waiting; i += 50_000) await take(await client.lrange(`${prefix}:wait`, i, i + 49_999), INSPECTION_PRIORITY.normal)
+      // Prioritized jobs were the low-priority niconico captures.
+      const prioritized = await client.zcard(`${prefix}:prioritized`)
+      for (let i = 0; i < prioritized; i += 50_000) await take(await client.zrange(`${prefix}:prioritized`, i, i + 49_999), INSPECTION_PRIORITY.low)
+      this.logger.info(`${this.dryRun ? 'Would move' : 'Moved'} ${moved} jobs into inspection_queue (${skipped} with other ids skipped).`)
+      if (this.dryRun) return
+      // Waiting and prioritized jobs only; active ones finish as they are.
+      await queue.drain()
+      const counts = await queue.getJobCounts('wait', 'prioritized', 'active')
+      this.logger.info(`BullMQ now: ${counts.wait} waiting, ${counts.prioritized} prioritized, ${counts.active} active; table: ${await InspectionQueue.size()}.`)
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+}

commands/ReleaseRebuild.ts

@@ -70,7 +70,7 @@ export default class ReleaseRebuild extends BaseCommand {
                AND NOT (v.release_id IS NULL
                         AND b.release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))
                -- A posted NZB decided this grouping (NzbImportService); name rules don't override it.
-               AND NOT EXISTS (SELECT 1 FROM files n WHERE n.release_id = b.release_id AND n.content_type = 'nzb')`,
+               AND NOT EXISTS (SELECT 1 FROM files n JOIN file_details nd ON nd.file_id = n.id WHERE n.release_id = b.release_id AND nd.content_type = 'nzb')`,
             [ids, releaseIdsPerBinary] as any
           )
         })
@@ -85,12 +85,13 @@ export default class ReleaseRebuild extends BaseCommand {
         const rows = await db
           .from('files as f')
           .join('binaries as b', 'b.file_id', 'f.id')
-          .whereNotNull('f.archive')
+          .join('file_details as d', 'd.file_id', 'f.id')
+          .whereNotNull('d.archive')
           .whereNull('b.release_id')
           .where('f.id', '>', rarCursor)
           .orderBy('f.id')
           .limit(BATCH)
-          .select('f.id as file_id', 'f.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
+          .select('f.id as file_id', 'd.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
         if (rows.length === 0) break
         rarCursor = Number(rows.at(-1).file_id)
         const segments = rows.map((r) => ({
@@ -147,7 +148,8 @@ export default class ReleaseRebuild extends BaseCommand {
     await db.transaction(async (trx) => {
       await trx.rawQuery('DELETE FROM release_files')
       await trx.rawQuery(`UPDATE binaries SET real_name = NULL WHERE real_name IS NOT NULL`)
-      await trx.rawQuery(`UPDATE files SET real_name = NULL, archive = NULL WHERE real_name IS NOT NULL OR archive IS NOT NULL`)
+      await trx.rawQuery(`UPDATE files SET real_name = NULL WHERE real_name IS NOT NULL`)
+      await trx.rawQuery(`UPDATE file_details SET archive = NULL WHERE archive IS NOT NULL`)
       await trx.rawQuery(`UPDATE releases SET contents = NULL, encrypted = NULL WHERE contents IS NOT NULL OR encrypted IS NOT NULL`)
       // Binaries whose name rules find no release were grouped from contents (or wrongly by size).
       await trx.rawQuery(`

commands/ReleaseRename.ts

@@ -1,6 +1,6 @@
-import { BaseCommand } from '@adonisjs/core/ace'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
-import { renameFromStoredContents } from '#services/ArchiveService'
+import { renameFromStoredContents, renameHiddenNamed } from '#services/ArchiveService'
 import { regroupAndRenameFromPar2 } from '#services/NameService'
 
 /** Renames random-named releases from their stored archive listings (no downloads). */
@@ -12,7 +12,14 @@ export default class ReleaseRename extends BaseCommand {
     startApp: true,
   }
 
+  @flags.boolean({ description: 'Only releases named after a hidden archive entry (e.g. a decoy ".x.mkv")' })
+  declare hidden: boolean
+
   public async run() {
+    if (this.hidden) {
+      this.logger.info(`Renamed ${await renameHiddenNamed()} releases named after a hidden archive entry.`)
+      return
+    }
     const par2 = await regroupAndRenameFromPar2()
     this.logger.info(`par2: ${par2.regrouped} files moved into the set that names them, ${par2.renamed} releases renamed.`)
     this.logger.info(`Renamed ${await renameFromStoredContents()} releases from their archive contents.`)

commands/SeedNode.ts

@@ -1,4 +1,6 @@
 import { gzipSync, unzipSync } from 'node:zlib'
+import { spawn, type ChildProcess } from 'node:child_process'
+import { availableParallelism } from 'node:os'
 import { setTimeout as sleep } from 'node:timers/promises'
 import { BaseCommand, flags } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
@@ -8,10 +10,18 @@ import { XoverClient, parseOverviews, type Overview } from '#lib/nntp/xover_clie
 import { runRange, type OverviewSource } from '#lib/seed/runner'
 import type { SeedBatch } from '#lib/seed/wire'
 import type { OverviewHeader } from '#lib/ingest/segment'
+import { probeInspection, probeWalk, type ContentTask } from '#lib/content/probe'
 
 class LeaseLost extends Error {}
 class Stopping extends Error {}
 
+interface InspectJob {
+  queue: string
+  jobId: string
+  token: string
+  task: ContentTask
+}
+
 interface Lease {
   id: number
   group: string
@@ -25,7 +35,7 @@ interface Lease {
  * connection(s) (D37), and posts the results back in gzip-compressed
  * batches. Outgoing requests only: no ports, database or Redis needed.
  *
- *   SEED_URL=https://seed.example SEED_TOKEN=sn_… NNTP_HOST=… node ace seed:node --connections 4
+ *   SEED_URL=https://seed.example SEED_TOKEN=sn_… node ace seed:node --ranges 4 --inspectors 2
  */
 export default class SeedNode extends BaseCommand {
   public static commandName = 'seed:node'
@@ -45,10 +55,17 @@ export default class SeedNode extends BaseCommand {
 
   @flags.number({
     description:
-      'Ranges worked on at once, one NNTP connection each (default: SEED_CONNECTIONS or 1)',
-    default: Number(process.env.SEED_CONNECTIONS) || 1,
+      'Ranges seeded at once (with SEED_FETCH=direct, one NNTP connection each); 0 for an inspection-only node (default: SEED_RANGES or 1)',
+    default: process.env.SEED_RANGES ? Number(process.env.SEED_RANGES) : 1,
   })
-  declare connections: number
+  declare ranges: number
+
+  @flags.number({
+    description:
+      'Content inspections and archive walks worked on at once, articles through the seed API (default: SEED_INSPECTORS or 0)',
+    default: Number(process.env.SEED_INSPECTORS) || 0,
+  })
+  declare inspectors: number
 
   @flags.number({ description: 'Articles per XOVER', default: 20_000 })
   declare chunk: number
@@ -63,10 +80,19 @@ export default class SeedNode extends BaseCommand {
   })
   declare fetch: string
 
+  @flags.number({
+    description:
+      'Processes the ranges and inspections are spread over; one process uses one core (default: SEED_PROCESSES or the number of cores)',
+    default: process.env.SEED_PROCESSES ? Number(process.env.SEED_PROCESSES) : availableParallelism(),
+  })
+  declare processes: number
+
   @flags.boolean({ description: 'Stop when there is nothing to lease instead of waiting' })
   declare once: boolean
 
   private stopping = false
+  /** In a worker process: its number, in front of its log lines. */
+  private readonly child = process.env.SEED_PROCESS ?? ''
 
   public async run() {
     const base = (this.url ?? process.env.SEED_URL ?? '').replace(/\/+$/, '')
@@ -76,23 +102,82 @@ export default class SeedNode extends BaseCommand {
       this.exitCode = 1
       return
     }
+    // JavaScript runs on one core per process: several slots are spread over worker processes.
+    const processes = Math.max(1, Math.min(this.processes || 1, this.ranges + this.inspectors))
+    if (!this.child && processes > 1) return this.supervise(base, token, processes)
+
     const stop = () => {
-      if (!this.stopping) this.logger.info('Stopping: giving back the ranges in progress…')
+      if (!this.stopping && !this.child) this.logger.info('Stopping: giving back the ranges in progress…')
       this.stopping = true
     }
     process.on('SIGTERM', stop)
     process.on('SIGINT', stop)
 
     const api = new SeedApi(base, token, (message) => this.logger.warning(message))
-    this.logger.info(
-      `Seeding node for ${base}: ${this.connections} range(s) at once, ${this.fetch === 'direct' ? 'own NNTP connections' : 'headers through the seed API'}.`
+    if (!this.child) this.logger.info(
+      `Seeding node for ${base}: ${this.ranges} range(s) at once, ${this.fetch === 'direct' ? 'own NNTP connections' : 'headers through the seed API'}` +
+        (this.inspectors > 0 ? `; ${this.inspectors} inspection(s) at once, articles through the seed API.` : '.')
     )
-    await Promise.all(Array.from({ length: this.connections }, (_, i) => this.work(api, i + 1)))
+    await Promise.all([
+      ...Array.from({ length: this.ranges }, (_, i) => this.work(api, i + 1)),
+      ...Array.from({ length: this.inspectors }, (_, i) => this.inspect(api, i + 1)),
+    ])
+    if (!this.child) this.logger.info('Seeding node stopped.')
+  }
+
+  /**
+   * Runs the slots in `processes` worker processes (this command again, with
+   * SEED_PROCESS set), each with its share of the ranges and inspections. A
+   * worker that exits while the node runs is restarted; SIGTERM/SIGINT are
+   * passed on, and the node stops when all workers have given their work back.
+   */
+  private async supervise(base: string, token: string, processes: number) {
+    // Slots dealt out in turn (ranges first, then inspections), so every process has work.
+    const share = (from: number, to: number, i: number) =>
+      Array.from({ length: to - from }, (_, k) => from + k).filter((k) => k % processes === i).length
+    this.logger.info(
+      `Seeding node for ${base}: ${this.ranges} range(s) and ${this.inspectors} inspection(s) at once, over ${processes} processes, ` +
+        `${this.fetch === 'direct' ? 'own NNTP connections' : 'articles and headers through the seed API'}.`
+    )
+    const children = new Set<ChildProcess>()
+    const start = (n: number): Promise<void> =>
+      new Promise((resolve) => {
+        const args = [
+          ...process.execArgv,
+          process.argv[1],
+          'seed:node',
+          `--ranges=${share(0, this.ranges, n - 1)}`,
+          `--inspectors=${share(this.ranges, this.ranges + this.inspectors, n - 1)}`,
+          `--fetch=${this.fetch}`,
+          `--chunk=${this.chunk}`,
+          `--submit-every=${this.submitEvery}`,
+          ...(this.once ? ['--once'] : []),
+        ]
+        const child = spawn(process.execPath, args, {
+          env: { ...process.env, SEED_URL: base, SEED_TOKEN: token, SEED_PROCESS: String(n) },
+          stdio: 'inherit',
+        })
+        children.add(child)
+        child.on('exit', (code, signal) => {
+          children.delete(child)
+          if (this.stopping || this.once || code === 0) return resolve()
+          this.logger.warning(`Process ${n} exited (${signal ?? code}); restarting in 10 s`)
+          setTimeout(() => start(n).then(resolve), 10_000)
+        })
+      })
+    const stop = (signal: NodeJS.Signals) => {
+      if (!this.stopping) this.logger.info('Stopping: giving back the ranges and inspections in progress…')
+      this.stopping = true
+      for (const child of children) child.kill(signal)
+    }
+    process.on('SIGTERM', () => stop('SIGTERM'))
+    process.on('SIGINT', () => stop('SIGINT'))
+    await Promise.all(Array.from({ length: processes }, (_, i) => start(i + 1)))
     this.logger.info('Seeding node stopped.')
   }
 
   private async work(api: SeedApi, worker: number) {
-    const log = (message: string) => this.logger.info(`[${worker}] ${message}`)
+    const log = (message: string) => this.logger.info(`[${this.child && `${this.child}/`}${worker}] ${message}`)
     // The NNTP connection is opened for a lease and kept while leases keep coming; with
     // nothing to lease it's closed, so an idle node holds no connections.
     let client: XoverClient | undefined
@@ -137,6 +222,50 @@ export default class SeedNode extends BaseCommand {
     }
   }
 
+  /**
+   * Content inspection and archive walks (CONCEPT.md D41): lease a few jobs,
+   * probe each with articles through the seed API, post the results. The
+   * indexer applies them; a job left unanswered goes back after 10 minutes.
+   */
+  private async inspect(api: SeedApi, slot: number) {
+    const log = (message: string) => this.logger.info(`[inspect ${this.child && `${this.child}/`}${slot}] ${message}`)
+    const fetchArticle = (id: string) => api.get(`/api/v1/seed/articles?id=${encodeURIComponent(id)}`, 4)
+    while (!this.stopping) {
+      let jobs: InspectJob[] = []
+      try {
+        jobs = ((await api.post('/api/v1/seed/inspect/lease', { max: 2 })) as { jobs: InspectJob[] } | null)?.jobs ?? []
+      } catch (error: any) {
+        if (error instanceof Unauthorized) throw error
+        log(`Lease failed: ${error.message}`)
+      }
+      if (jobs.length === 0) {
+        if (this.once) return
+        await this.pause(30_000)
+        continue
+      }
+      for (const job of jobs) {
+        const ref = { queue: job.queue, jobId: job.jobId, token: job.token }
+        if (this.stopping) {
+          await api.post('/api/v1/seed/inspect/release', ref).catch(() => {})
+          continue
+        }
+        try {
+          const result =
+            job.task.kind === 'inspect'
+              ? await probeInspection(job.task, fetchArticle, (message) => log(message))
+              : await probeWalk(job.task, fetchArticle)
+          await api.post('/api/v1/seed/inspect/done', { ...ref, result: JSON.stringify(result) })
+        } catch (error: any) {
+          if (error instanceof Unauthorized) throw error
+          // The job's lock expired and it went to another node: nothing to report.
+          if (error instanceof LeaseLost) continue
+          log(`${job.task.kind === 'inspect' ? `"${job.task.name}"` : `walk of release ${job.task.releaseId}`} failed: ${error.message}`)
+          await api.post('/api/v1/seed/inspect/failed', { ...ref, error: error.message }).catch(() => {})
+        }
+      }
+    }
+  }
+
   private connect() {
     return XoverClient.connect({
       host: nntpConfig.host,
@@ -172,6 +301,8 @@ export default class SeedNode extends BaseCommand {
     const started = Date.now()
     let headers = 0
     let chain: Promise<void> = Promise.resolve()
+    // Where the range resumes after what was stored: a batch split in two reports it with its first half.
+    let reported = lease.start
     const posting = new Set<Promise<void>>()
     // Throws once a post has failed, so the range stops fetching.
     const failedSoFar = () => Promise.race([chain, Promise.resolve()])
@@ -192,7 +323,8 @@ export default class SeedNode extends BaseCommand {
           if (this.stopping) throw new Stopping()
           while (posting.size >= 2) await Promise.race(posting)
           const post = chain.then(async () => {
-            await api.post(`/api/v1/seed/ranges/${lease.id}/batches`, batch)
+            await this.postBatch(api, lease.id, batch, reported)
+            reported = batch.next
             headers += batch.headers
           })
           chain = post
@@ -225,6 +357,26 @@ export default class SeedNode extends BaseCommand {
     }
   }
 
+  /**
+   * Posts a batch; one the seed API finds too large (413: dense recent ranges) is
+   * posted in two halves, at most three splits deep. Only the second half moves
+   * the range's resume point, so a failure in between loses nothing.
+   */
+  private async postBatch(api: SeedApi, rangeId: number, batch: SeedBatch, reported: number, depth = 0): Promise<void> {
+    try {
+      await api.post(`/api/v1/seed/ranges/${rangeId}/batches`, batch)
+    } catch (error: any) {
+      const parts = batch.files.length + batch.fragments.length
+      if (!/: 413 /.test(String(error.message)) || depth >= 3 || parts < 2) throw error
+      const files = Math.ceil(batch.files.length / 2)
+      const fragments = Math.ceil(batch.fragments.length / 2)
+      const first: SeedBatch = { ...batch, next: reported, headers: 0, obfuscated: 0, deferred: 0, files: batch.files.slice(0, files), fragments: batch.fragments.slice(0, fragments) }
+      const second: SeedBatch = { ...batch, files: batch.files.slice(files), fragments: batch.fragments.slice(fragments) }
+      await this.postBatch(api, rangeId, first, reported, depth + 1)
+      await this.postBatch(api, rangeId, second, reported, depth + 1)
+    }
+  }
+
   /** Sleeps, but wakes up early when the node is stopping. */
   private async pause(ms: number) {
     for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)
@@ -249,12 +401,12 @@ class SeedApi {
     return text ? JSON.parse(text) : null
   }
 
-  /** A binary GET (a relay chunk). */
-  async get(path: string): Promise<Buffer> {
-    return (await this.request('GET', path)) ?? Buffer.alloc(0)
+  /** A binary GET (a relay chunk, an article). */
+  async get(path: string, attempts = 14): Promise<Buffer> {
+    return (await this.request('GET', path, undefined, attempts)) ?? Buffer.alloc(0)
   }
 
-  private async request(method: string, path: string, payload?: Buffer): Promise<Buffer | null> {
+  private async request(method: string, path: string, payload?: Buffer, attempts = 14): Promise<Buffer | null> {
     for (let attempt = 1; ; attempt++) {
       let status = 0
       let data = Buffer.alloc(0)
@@ -283,7 +435,7 @@ class SeedApi {
         throw new Error(`Seed API ${path}: ${status} ${text()}`)
       // Network errors, 404 and 5xx: storing a batch twice is harmless, so try again.
       // ~20 minutes in all, longer than a lease lasts: a restart or maintenance of the API is waited out.
-      if (attempt >= 14) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text()}`)
+      if (attempt >= attempts) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text()}`)
       const wait = Math.min(120_000, 2000 * 2 ** (attempt - 1))
       this.warn(`Seed API ${path}: ${status || text()}; retrying in ${wait / 1000} s`)
       await sleep(wait)

commands/SeedPlan.ts

@@ -4,9 +4,10 @@ import NntpService from '#services/NntpService'
 import SeedOrchestrator from '#services/SeedOrchestrator'
 
 /**
- * Plans seed ranges (CONCEPT.md D38) for groups, from below what live
- * indexing, the backfill and earlier plans cover down to `--days` back (or
- * the server's oldest article). A group's backfill stops where its plan
+ * Plans seed ranges (CONCEPT.md D38) for groups: every article from below
+ * what live indexing and the backfill cover down to `--days` back (or the
+ * server's oldest article) that no earlier range covers. Running it again
+ * plans only what's missing. A group's backfill stops where its plan
  * starts. Ranges are interleaved across groups, newest first.
  *
  *   node ace seed:plan alt.binaries.ath alt.binaries.mom --days 365
@@ -28,11 +29,16 @@ export default class SeedPlan extends BaseCommand {
   @flags.number({ description: 'Articles per range', default: 2_000_000 })
   declare size: number
 
+  @flags.number({ description: 'Leased before ranges of a lower priority (default 0)', default: 0 })
+  declare priority: number
+
   public async run() {
     try {
-      const summary = await SeedOrchestrator.plan(this.groups, { days: this.days, size: this.size })
+      const summary = await SeedOrchestrator.plan(this.groups, { days: this.days, size: this.size, priority: this.priority })
       for (const s of summary) {
-        this.logger.info(`${s.group}: ${s.ranges.toLocaleString('en')} ranges, articles ${s.from.toLocaleString('en')}–${s.to.toLocaleString('en')}.`)
+        this.logger.info(
+          `${s.group}: ${s.ranges.toLocaleString('en')} new ranges in ${s.gaps} gap(s) not yet planned, articles ${s.from.toLocaleString('en')}–${s.to.toLocaleString('en')}.`
+        )
       }
     } finally {
       await NntpService.shutdown()

commands/SeedRange.ts

@@ -51,7 +51,7 @@ export default class SeedRange extends BaseCommand {
     const first = Number(this.start)
     if (!Number.isSafeInteger(first) || first < 1) throw new Error(`--start must be an article number, got ${this.start}`)
     const last = first + this.count - 1
-    const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, fragments: 0, inspections: 0 }
+    const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, dropped: 0, fragments: 0, inspections: 0 }
     let storeMs = 0
     const cpuStart = process.cpuUsage()
     const started = Date.now()
@@ -86,10 +86,11 @@ export default class SeedRange extends BaseCommand {
           if (!this.dryRun) {
             const t = Date.now()
             const stored = await SeedService.store(batch.files)
-            await SeedService.storeFragments(this.group, batch.fragments)
+            await SeedService.storeFragments(batch.fragments, { start: first, end: last })
             storeMs += Date.now() - t
             stats.files += stored.files
             stats.existing += stored.existing
+            stats.dropped += stored.dropped
             stats.inspections += stored.inspections
           }
           this.progress(stats, started)
@@ -106,7 +107,7 @@ export default class SeedRange extends BaseCommand {
     this.logger.info(
       `${this.group} ${first}-${last}: ${n(stats.headers)} headers in ${seconds.toFixed(0)} s (${n(Math.round(stats.headers / seconds))}/s). ` +
         `${n(stats.obfuscated)} obfuscated, ${n(stats.deferred)} deferred. ` +
-        `${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known), ` +
+        `${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known, ${n(stats.dropped)} garbage), ` +
         `${n(stats.fragments)} fragment segments, ${n(stats.inspections)} queued for inspection.`
     )
     this.logger.info(`Store ${(storeMs / 1000).toFixed(0)} s; node CPU ${((cpu.user + cpu.system) / 1e6).toFixed(0)} s.`)

commands/SpotPostprocess.ts

@@ -0,0 +1,55 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+const BATCH = 500
+
+/**
+ * Reads the article and NZB of listed spots that were listed from their
+ * overview only (LOG.md 2026-10-08 (10)): description, the NZB's password or
+ * one in the spot's text, placeholder NZBs. Newest first; can be stopped and
+ * started again (worker:spots does the same when idle, a batch at a time).
+ * Connections: NNTP_POOL_SIZE of this process, --concurrency at a time.
+ */
+export default class SpotPostprocess extends BaseCommand {
+  public static commandName = 'spot:postprocess'
+  public static description = "Reads listed spots' articles and NZBs for their passwords and details."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Spots read at a time (default: NNTP_POOL_SIZE)' })
+  declare concurrency?: number
+
+  @flags.number({ description: 'Stop after this many spots' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = this.concurrency ?? env.get('NNTP_POOL_SIZE', 4)
+    const [{ count }] = await db.from('spots').where('status', 'listed').whereNull('nzb_read_at').count('* as count')
+    this.logger.info(`${count} listed spots to read, ${concurrency} at a time.`)
+    const totals: Record<string, number> = {}
+    const started = Date.now()
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const counts = await SpotService.readListings(Math.min(BATCH, this.limit ? this.limit - done : BATCH), concurrency)
+        const read = Object.values(counts).reduce((sum, n) => sum + n, 0)
+        if (read === 0) break
+        for (const [k, v] of Object.entries(counts)) totals[k] = (totals[k] ?? 0) + v
+        done += read
+        const rate = done / ((Date.now() - started) / 1000)
+        this.logger.info(`${done} read (${rate.toFixed(1)}/s): ${Object.entries(totals).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+        // Usenet unreachable: everything was given back; don't spin on it.
+        if (counts.retry === read) await Shutdown.sleep(30_000)
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

commands/workers/Collection.ts

@@ -6,7 +6,8 @@ import env from '#start/env'
 import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
 import NntpService from '#services/NntpService'
-import ContentService from '#services/ContentService'
+import ContentService, { describeContent } from '#services/ContentService'
+import ArchiveWalkService from '#services/ArchiveWalkService'
 import File from '#models/file'
 
 export default class CollectionWorker extends BaseCommand {
@@ -32,31 +33,7 @@ export default class CollectionWorker extends BaseCommand {
 
       try {
         const content = await ContentService.inspect(fileId)
-        if (content?.par2) {
-          const { par2 } = content
-          this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
-        } else if (content?.archive) {
-          const { info, releaseRenamed, grouped } = content.archive
-          const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
-            .filter(Boolean)
-            .join(', ')
-          this.logger.info(
-            `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
-              (grouped ? ', grouped into a release' : '') +
-              (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
-          )
-        }
-        if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
-        if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
-        if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
-        if (content?.yencName) this.logger.info(`"${file.filename}" is "${content.yencName}" (yEnc name).`)
-        if (content?.nicovideo) this.logger.info(`"${file.filename}" → "${content.nicovideo}".`)
-        if (content?.sfvMoved) this.logger.info(`SFV "${file.filename}" moved ${content.sfvMoved} files into its release.`)
-        if (content?.contentType === 'yencbin') this.logger.info(`"${file.filename}" is a yEncBin encrypted container.`)
-        if (content?.nzb) {
-          const n = content.nzb
-          this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
-        }
+        for (const line of describeContent(file.filename, content)) this.logger.info(line)
       } catch (error: any) {
         this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
       }
@@ -70,10 +47,28 @@ export default class CollectionWorker extends BaseCommand {
       this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
     })
 
+    // Complete archive listings (D40): background work beside inspection, on the same NNTP pool.
+    const walkConcurrency = env.get('ARCHIVE_WALK_CONCURRENCY', 1)
+    const walker = walkConcurrency > 0
+      ? new Worker('archive-walk-queue', async (job) => {
+          const o = await ArchiveWalkService.walk(Number(job.data.releaseId))
+          if (o.skipped) return o
+          this.logger.info(
+            `Archive walk of release ${o.releaseId}: ${o.before} → ${o.entries} entries from ${o.segments} segments` +
+              (o.complete ? '' : ` (stopped: ${o.stopped})`) +
+              (o.renamed ? `, renamed to "${o.renamed}"` : '')
+          )
+          return o
+        }, { connection: queueConfig.connection, concurrency: walkConcurrency })
+      : null
+    walker?.on('failed', (job, err) => {
+      this.logger.error(`Archive walk job ${job?.id} failed: ${err.message}`)
+    })
+
     this.logger.info('Collection worker started and listening for jobs.')
     await Shutdown.wait()
     this.logger.info('Shutting down: finishing active jobs...')
-    await worker.close()
+    await Promise.all([worker.close(), walker?.close()])
     await NntpService.shutdown()
     await QueueService.closeAll()
   }

commands/workers/Deferred.ts

@@ -31,23 +31,29 @@ export default class DeferredWorker extends BaseCommand {
     let done = 0
     try {
       while (!Shutdown.requested && (!this.limit || done < this.limit)) {
-        const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
-        if (batch.length === 0) {
-          if (this.limit) break
-          await Shutdown.sleep(IDLE_MS)
-          continue
+        try {
+          const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+          if (batch.length === 0) {
+            if (this.limit) break
+            await Shutdown.sleep(IDLE_MS)
+            continue
+          }
+          let next = 0
+          await Promise.all(
+            Array.from({ length: concurrency }, async () => {
+              while (next < batch.length) {
+                const status = await DeferredService.process(batch[next++])
+                counts[status] = (counts[status] ?? 0) + 1
+              }
+            })
+          )
+          done += batch.length
+          this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+        } catch (err: any) {
+          // E.g. Postgres restarting: wait and claim again, instead of ending the loop (LOG.md 2026-10-07 (3)).
+          this.logger.error(`Round failed, retrying in 30 s: ${err.message}`)
+          await Shutdown.sleep(30_000)
         }
-        let next = 0
-        await Promise.all(
-          Array.from({ length: concurrency }, async () => {
-            while (next < batch.length) {
-              const status = await DeferredService.process(batch[next++])
-              counts[status] = (counts[status] ?? 0) + 1
-            }
-          })
-        )
-        done += batch.length
-        this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
       }
     } finally {
       await NntpService.shutdown()

commands/workers/File.ts

@@ -1,14 +1,11 @@
 import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
-import db from '@adonisjs/lucid/services/db'
 import env from '#start/env'
-import FileStore from '#services/FileStore'
-import SfvService from '#services/SfvService'
+import BinaryFileService from '#services/BinaryFileService'
 import queueConfig from '#config/queue'
 import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
-import ContentService from '#services/ContentService'
 
 export default class FileWorker extends BaseCommand {
   public static commandName = 'worker:file'
@@ -21,83 +18,21 @@ export default class FileWorker extends BaseCommand {
   public async run() {
     this.logger.info('Starting file worker...')
 
-    const collectionQueue = QueueService.collectionQueue
-
     const worker = new Worker('file-queue', async (job) => {
-      const { binaryId } = job.data
-
-      const binary = await db.from('binaries').where('id', binaryId).first()
-      if (!binary) {
-        this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`)
-        return
+      // A batch of completed binaries ('process-files'), or one from before batching ('process-file').
+      const ids: number[] = (job.data.binaryIds ?? [job.data.binaryId]).map(Number)
+      const result = await BinaryFileService.store(ids)
+      if (result.saved || result.existing) {
+        this.logger.info(
+          `Files: ${result.saved} saved, ${result.existing} existed already` +
+            (result.skipped ? `, ${result.skipped} skipped` : '') +
+            (result.inspections ? `, ${result.inspections} queued for inspection` : '') +
+            ` (${ids.length} binaries).`
+        )
       }
-
-      if (binary.conflicted_at) {
-        this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`)
-        return
-      }
-
-      const parts = await db
-        .from('parts')
-        .where('binary_id', binaryId)
-        .orderBy('number')
-        .select('number', 'message_id', 'bytes')
-      if (parts.length < binary.total_parts) {
-        throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
-      }
-
-      const segments = parts.map((part) => ({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) }))
-
-      // Already known as a file, e.g. added from a posted NZB before its own articles completed.
-      const existing = await db.from('files').where('first_message_id', segments[0]?.id ?? '').first()
-      if (existing) {
-        await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
-        await dropCopiedParts(binary)
-        this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`)
-        return
-      }
-
-      // file_key is unique: a second completion of the same upload inserts nothing.
-      const now = new Date()
-      const [inserted] = await FileStore.insert([
-        {
-          file_key: binary.file_key,
-          filename: binary.name,
-          poster: binary.poster,
-          date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
-          parts: parts.length,
-          segments,
-          groups: JSON.stringify(binary.groups),
-          release_id: binary.release_id,
-          real_name: binary.real_name,
-          size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
-          created_at: now,
-          updated_at: now,
-        },
-      ])
-
-      const fileId =
-        inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
-      await db.from('binaries').where('id', binaryId).update({ file_id: fileId })
-      await dropCopiedParts(binary)
-
-      if (!inserted) {
-        this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`)
-        return
-      }
-
-      this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
-      // An SFV posted earlier may already list it (D34).
-      await SfvService.claim({
-        id: Number(fileId),
-        name: binary.real_name ?? binary.name,
-        poster: binary.poster,
-        date: binary.posted_at ? new Date(binary.posted_at).getTime() : null,
-        release_id: binary.release_id,
-      })
-      // Files inspection would skip anyway (later volumes, readable media names) don't take a queue slot.
-      if (ContentService.isCandidate(binary.name)) {
-        await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
+      // The rest is stored; the job's retry picks these up (the others are linked now and skipped).
+      if (result.incomplete.length > 0) {
+        throw new Error(`${result.incomplete.length} of ${ids.length} binaries don't have all their parts yet.`)
       }
     }, {
       connection: queueConfig.connection,
@@ -117,20 +52,3 @@ export default class FileWorker extends BaseCommand {
     await QueueService.closeAll()
   }
 }
-
-/**
- * The file holds every message id now. Binaries identified by their subject
- * (no size: header-derived) don't need their parts any more; body-derived
- * ones keep them, so articles of an equal-sized next volume are recognised
- * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
- */
-/**
- * The file holds the segment list now. Header-derived parts (no per-part
- * CRC) aren't needed any more; body-derived ones are kept for split-volume
- * verification (D20). Going by the CRC, not the binary's size: content
- * inspection gives header-derived binaries a size too, which kept ~20M
- * parts around (LOG.md 2026-10-02).
- */
-async function dropCopiedParts(binary: { id: number }) {
-  await db.from('parts').where('binary_id', binary.id).whereNull('pcrc32').delete()
-}

commands/workers/Spots.ts

@@ -10,10 +10,13 @@ const IDLE_MS = 30_000
 const BATCH = 500
 /** Imported spots whose details are read per idle round. */
 const DETAILS_BATCH = 50
+/** Listed spots whose article and NZB are read per idle round (passwords, dummies). */
+const LISTINGS_BATCH = 50
 
 /**
  * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
- * downloads its compressed NZB (a few KB to a few MB) and imports it.
+ * downloads its compressed NZB (a few KB to a few MB) and imports it. When
+ * idle, reads what listed spots were listed without: details, passwords.
  */
 export default class SpotsWorker extends BaseCommand {
   public static commandName = 'worker:spots'
@@ -33,28 +36,38 @@ export default class SpotsWorker extends BaseCommand {
     let done = 0
     try {
       while (!Shutdown.requested && (!this.limit || done < this.limit)) {
-        const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
-        if (batch.length === 0) {
-          if (this.limit) break
-          // Idle: read the details of imported spots that don't have them yet (newest first).
-          if ((await SpotService.fillDetails(DETAILS_BATCH)) === 0) await Shutdown.sleep(IDLE_MS)
-          continue
+        try {
+          const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
+          if (batch.length === 0) {
+            if (this.limit) break
+            // Idle: read the details of imported spots that don't have them yet (newest first), then
+            // the article and NZB of listed spots (their passwords; LOG.md 2026-10-08 (10)).
+            if ((await SpotService.fillDetails(DETAILS_BATCH)) > 0) continue
+            const listings = await SpotService.readListings(LISTINGS_BATCH, concurrency)
+            const read = Object.values(listings).reduce((sum, n) => sum + n, 0)
+            if (read === 0 || listings.retry === read) await Shutdown.sleep(IDLE_MS)
+            continue
+          }
+          // Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
+          const listable = batch.filter((spot) => SpotService.isListable(spot))
+          const single = batch.filter((spot) => !SpotService.isListable(spot))
+          counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
+          let next = 0
+          await Promise.all(
+            Array.from({ length: concurrency }, async () => {
+              while (next < single.length) {
+                const status = await SpotService.process(single[next++])
+                counts[status] = (counts[status] ?? 0) + 1
+              }
+            })
+          )
+          done += batch.length
+          this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+        } catch (err: any) {
+          // E.g. Postgres restarting: wait and claim again, instead of ending the loop (LOG.md 2026-10-07 (3)).
+          this.logger.error(`Round failed, retrying in 30 s: ${err.message}`)
+          await Shutdown.sleep(30_000)
         }
-        // Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
-        const listable = batch.filter((spot) => SpotService.isListable(spot))
-        const single = batch.filter((spot) => !SpotService.isListable(spot))
-        counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
-        let next = 0
-        await Promise.all(
-          Array.from({ length: concurrency }, async () => {
-            while (next < single.length) {
-              const status = await SpotService.process(single[next++])
-              counts[status] = (counts[status] ?? 0) + 1
-            }
-          })
-        )
-        done += batch.length
-        this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
       }
     } finally {
       await NntpService.shutdown()

config/gc.ts

@@ -7,6 +7,15 @@ export default {
   /** How often `gc:scheduler` sweeps. */
   intervalMs: env.get('GC_INTERVAL_MS', 3 * 60 * 1000),
 
+  /** How often gc:scheduler recomputes the completeness of releases that changed (LOG.md 2026-10-06 (6)). */
+  releaseCompleteIntervalMs: env.get('RELEASE_COMPLETE_INTERVAL_MS', 30 * 60 * 1000),
+
+  /** How often gc:scheduler merges uploads split per poster (D35); it reads every release. */
+  mergeIntervalMs: env.get('GC_MERGE_INTERVAL_MS', 60 * 60 * 1000),
+
+  /** Time per sweep for copying old file keys into file_keys until it's complete (LOG.md 2026-10-06 (7)). */
+  fileKeysFillMs: env.get('FILE_KEYS_FILL_SECONDS', 90) * 1000,
+
   jobs: {
     /** Completed jobs are only kept for inspection; keep them briefly. */
     completedAgeSeconds: env.get('GC_COMPLETED_AGE_SECONDS', 60 * 60),
@@ -40,6 +49,11 @@ export default {
      * later, so they go sooner than staleAfterSeconds.
      */
     unfingerprintedAfterSeconds: env.get('GC_UNFINGERPRINTED_SECONDS', 6 * 60 * 60),
+    /**
+     * A finished binary whose file waits for content inspection is kept this long at
+     * most (it needs the fingerprint inspection stores for par2 naming, D16).
+     */
+    awaitInspectionSeconds: env.get('GC_AWAIT_INSPECTION_SECONDS', 7 * 24 * 60 * 60),
   },
 
   /**

config/indexer.ts

@@ -66,6 +66,19 @@ export default {
   seed: {
     /** NNTP connections the seed API fetches with for relay nodes (CONCEPT.md D38); 0: relay off. */
     nntpConnections: env.get('SEED_NNTP_CONNECTIONS', 0),
+    /**
+     * Silence that counts as a dead connection during a relay XOVER (an
+     * inactivity timeout: slow ranges keep sending). Chunks that "never
+     * answered" in 10 minutes came in 0.3 s on a fresh connection.
+     */
+    nntpTimeoutMs: env.get('SEED_NNTP_TIMEOUT_MS', 90 * 1000),
+    /**
+     * Ask the news server for gzip-compressed overviews (SEED_NNTP_COMPRESS=false: plain).
+     * On one connection plain had a shorter tail for 2026 articles, but with 20 relay
+     * connections both gave ~1.5 chunks/s, plain at many times the relay's CPU and
+     * more traffic (LOG.md 2026-10-06 (9)).
+     */
+    nntpCompress: env.get('SEED_NNTP_COMPRESS', true),
   },
 
   backfill: {
@@ -82,6 +95,25 @@ export default {
     maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
   },
 
+  /**
+   * Dropping garbage uploads early (CONCEPT.md D42). A signature (posting tool, first
+   * group) is dropped once at least minProbes of its probed random-named releases are
+   * known and at most maxNamedRate of them got a real name or password; its
+   * random-named files are then not stored, except explorePercent of its releases.
+   */
+  garbage: {
+    /** Off: nothing is filtered or deleted (the learning still runs). */
+    enabled: env.get('GARBAGE_FILTER', true),
+    explorePercent: env.get('GARBAGE_EXPLORE_PERCENT', 2),
+    minProbes: 100,
+    maxNamedRate: 0.005,
+    /** Probed releases learned from: 1 in this many (by release id), for the last windowDays. */
+    probeSample: 4,
+    windowDays: 30,
+    /** How often gc:scheduler learns the signatures again. */
+    learnIntervalMs: env.get('GARBAGE_LEARN_INTERVAL_MS', 3 * 60 * 60 * 1000),
+  },
+
   /** worker:deferred leaves a file alone until it has had no new articles for this long. */
   deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
   /** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */

database/migrations/1768620765160_files_bigint.ts

@@ -0,0 +1,89 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+const TRIGGERS = [
+  `files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`,
+  `files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`,
+  `files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`,
+  `files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`,
+]
+
+/**
+ * files.id (and binaries.file_id, the id sequence) to 64 bits (CONCEPT.md
+ * D39): seeding adds ~14M files a day, and 32 bits run out at 2.1 billion.
+ * Postgres can't change the type of a partition key, so the partitions are
+ * detached, rewritten one by one as plain tables (files_p0: ~100 GB with its
+ * segment lists, 1–2 hours on homelab's HDD) and attached to a new 64-bit
+ * parent. A CHECK per partition, added before its rewrite, spares the attach
+ * scan. Run with the writers stopped and Postgres on wal_level=minimal, or the
+ * rewrite also goes through the WAL (LOG.md 2026-10-05 (8)).
+ */
+export default class extends BaseSchema {
+  // One transaction: a failure anywhere leaves files as it was (it drops and recreates the parent).
+
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    const { rows: partitions } = await q(
+      `SELECT c.relname AS name, pg_get_expr(c.relpartbound, c.oid) AS bound
+       FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname`
+    )
+    const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`)
+    const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : ''
+
+    // The sequence belongs to the old parent: keep it when that goes.
+    await q(`ALTER SEQUENCE files_id_seq OWNED BY NONE`)
+    await q(`ALTER SEQUENCE files_id_seq AS bigint`)
+    await q(`ALTER TABLE binaries DROP CONSTRAINT IF EXISTS binaries_file_id_foreign`)
+    await q(`ALTER TABLE binaries ALTER COLUMN file_id TYPE bigint`)
+
+    for (const p of partitions) {
+      await q(`ALTER TABLE files DETACH PARTITION ${p.name}`)
+      // Triggers cloned from the parent may stay behind on a detached partition; the new parent clones them again.
+      for (const t of TRIGGERS) await q(`DROP TRIGGER IF EXISTS ${t.split(' ')[0]} ON ${p.name}`)
+      const [, from, to] = /FROM \((.+)\) TO \((.+)\)/.exec(p.bound.replaceAll("'", ''))!
+      const conditions = [from === 'MINVALUE' ? null : `id >= ${from}`, to === 'MAXVALUE' ? null : `id < ${to}`].filter(Boolean)
+      if (conditions.length > 0) await q(`ALTER TABLE ${p.name} ADD CONSTRAINT ${p.name}_bound CHECK (${conditions.join(' AND ')})`)
+    }
+    await q(`DROP TABLE files`)
+
+    for (const p of partitions) {
+      await q(`ALTER TABLE ${p.name} ALTER COLUMN id TYPE bigint`)
+    }
+
+    await q(`
+      CREATE TABLE files (
+        id bigint NOT NULL DEFAULT nextval('files_id_seq'),
+        filename varchar(255) NOT NULL,
+        poster varchar(255) NOT NULL,
+        date bigint NOT NULL,
+        parts integer NOT NULL,
+        groups jsonb,
+        created_at timestamptz,
+        updated_at timestamptz,
+        file_key varchar(40),
+        release_id bigint,
+        archive jsonb,
+        real_name text,
+        size bigint,
+        content_type varchar(16),
+        segments bytea NOT NULL,
+        first_message_id text
+      ) PARTITION BY RANGE (id)${tablespace}`)
+    await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`)
+    await q(`CREATE INDEX files_content_type_index ON files (content_type)`)
+    await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`)
+    await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`)
+    await q(`CREATE INDEX files_release_id_index ON files (release_id)`)
+    for (const p of partitions) {
+      await q(`ALTER TABLE files ATTACH PARTITION ${p.name} ${p.bound}`)
+      await q(`ALTER TABLE ${p.name} DROP CONSTRAINT IF EXISTS ${p.name}_bound`)
+    }
+    for (const t of TRIGGERS) await q(`CREATE TRIGGER ${t}`)
+    await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`)
+  }
+
+  async down() {
+    throw new Error('Not reversible: ids above 2^31 may exist.')
+  }
+}

database/migrations/1768620765170_release_contents_walked.ts

@@ -0,0 +1,14 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** When an archive walk (CONCEPT.md D40) looked at a release, so it isn't walked again. */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('releases', (table) => {
+      table.timestamp('contents_walked_at', { useTz: true })
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => table.dropColumn('contents_walked_at'))
+  }
+}

database/migrations/1768620765180_release_complete_faster.ts

@@ -0,0 +1,69 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Release completeness (D24) without cubic cost (LOG.md 2026-10-06 (4)). A
+ * par2 set listing n files was inserted into release_files row by row, each
+ * row's trigger ran release_complete() twice, and that matched every listed
+ * file against all binaries in the database: n × 2 × n × (all binaries). One
+ * insert of a few thousand rows ran for over 20 minutes at full CPU.
+ *
+ * - release_complete(): the release's files are read once, by release id,
+ *   and each listed file is matched against those by size or by name. The
+ *   correlated subquery read every binary in the database per listed file
+ *   (binaries_size_index), filtering by release only afterwards.
+ * - refresh_release_complete(): computes it once.
+ * - release_files: one refresh per release per INSERT statement.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
+        WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
+        -- The release's files, once, by index (read only when there's a par2 listing to check).
+        have AS MATERIALIZED (
+          SELECT b.size, f.real_name FROM files f JOIN binaries b ON b.file_id = f.id WHERE f.release_id = rid)
+        SELECT coalesce((
+          SELECT CASE
+            WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
+            WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
+              SELECT 1 FROM release_files rf
+              WHERE rf.release_id = rid AND rf.size >= 65536
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.real_name = rf.name))
+            ELSE false END
+          FROM r), false)
+      $$`)
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        UPDATE releases r SET complete = c.complete
+        FROM (SELECT release_complete(rid) AS complete) c
+        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
+      $$`)
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION release_files_release_complete_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          PERFORM refresh_release_complete(n.release_id) FROM (SELECT DISTINCT release_id FROM new_rows) n;
+          RETURN NULL;
+        END $$`)
+    // Swapping the trigger locks release_files: inserts still running the old cubic check
+    // (minutes) are cancelled rather than waited for; their jobs are retried, and run fast.
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
+    await this.db.rawQuery(`
+      SELECT pg_cancel_backend(pid) FROM pg_stat_activity
+      WHERE pid <> pg_backend_pid() AND state = 'active' AND query ILIKE 'insert into "release_files"%'
+        AND now() - query_start > interval '30 seconds'`)
+    await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
+    await this.db.rawQuery(`
+      CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
+      REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT
+      EXECUTE FUNCTION release_files_release_complete_statement()`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
+    await this.db.rawQuery(`
+      CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
+      FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
+    await this.db.rawQuery(`DROP FUNCTION IF EXISTS release_files_release_complete_statement()`)
+  }
+}

database/migrations/1768620765190_release_totals_per_statement.ts

@@ -0,0 +1,110 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Cheaper release bookkeeping for file inserts (LOG.md 2026-10-06 (6)).
+ *
+ * Every inserted file fired four row triggers, each updating its release:
+ * file_count/size, search_files, extensions and completeness, so a seed
+ * batch of thousands of files rewrote its releases thousands of times (a
+ * quarter of Postgres' active time on homelab). Now one statement-level
+ * trigger updates each release once per INSERT, with all four.
+ *
+ * Completeness (D24), the expensive part, is no longer computed in triggers:
+ * refresh_release_complete() only notes the release in release_dirty, and
+ * gc:scheduler recomputes the noted releases every 30 minutes
+ * (recompute_release_complete). Counts, sizes, search text and extensions
+ * stay current: code decides on them right away (weak releases, deleting
+ * empty releases).
+ *
+ * Also an index on release_files (release_id, size) for the par2 size
+ * match's "listed once in its set" check (IngestService.matchPar2Releases).
+ */
+export default class extends BaseSchema {
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    // Swapping triggers locks files and its partitions. Taken first and in one statement: a
+    // DROP TRIGGER later in the transaction deadlocked with ingest (2026-10-06). Give up rather
+    // than queue every writer behind it for long.
+    await q(`SET LOCAL lock_timeout = '2min'`)
+    await q(`LOCK TABLE files, release_files IN ACCESS EXCLUSIVE MODE`)
+    // Built CONCURRENTLY by hand on homelab (2026-10-06); IF NOT EXISTS keeps that one.
+    await q(`CREATE INDEX IF NOT EXISTS release_files_release_size_idx ON release_files (release_id, size)`)
+
+    // Append-only (no key): writers never wait on each other or on the sweep.
+    await q(`CREATE TABLE IF NOT EXISTS release_dirty (release_id bigint NOT NULL)`)
+    await q(`
+      CREATE OR REPLACE FUNCTION recompute_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        UPDATE releases r SET complete = c.complete
+        FROM (SELECT release_complete(rid) AS complete) c
+        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
+      $$`)
+    // Every trigger that called this (files, release_files, binaries.size, releases.file_total) now just notes the release.
+    await q(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL
+      $$`)
+
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          WITH added AS (
+            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
+                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
+                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
+            FROM (
+              SELECT release_id, size, name_extension(nm) AS e,
+                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
+              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
+            ) x
+            GROUP BY release_id
+          )
+          UPDATE releases r SET
+            file_count = r.file_count + a.n,
+            size = r.size + a.bytes,
+            extensions = CASE
+              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
+              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
+            search_files = CASE
+              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
+              ELSE nullif(left(concat_ws(' ', r.search_files, (
+                SELECT string_agg(w, ' ') FROM unnest(a.words) w
+                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
+          FROM added a
+          WHERE r.id = a.release_id;
+
+          INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
+          RETURN NULL;
+        END $$`)
+
+    // The row triggers keep UPDATE and DELETE (renames, moves between releases); INSERT is the statement trigger.
+    for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
+      await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
+    }
+    await q(`CREATE TRIGGER files_release_extension AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+    await q(`CREATE TRIGGER files_release_search AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+    await q(`CREATE TRIGGER files_release_totals AFTER DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
+    await q(`CREATE TRIGGER files_zz_release_complete AFTER DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
+    await q(`CREATE TRIGGER files_release_insert AFTER INSERT ON files REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION files_release_insert_statement()`)
+  }
+
+  async down() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`DROP TRIGGER IF EXISTS files_release_insert ON files`)
+    for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
+      await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
+    }
+    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+    await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
+    await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
+    await q(`DROP FUNCTION IF EXISTS files_release_insert_statement()`)
+    await q(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        UPDATE releases r SET complete = c.complete
+        FROM (SELECT release_complete(rid) AS complete) c
+        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
+      $$`)
+    await q(`DROP FUNCTION IF EXISTS recompute_release_complete(bigint)`)
+    await q(`DROP TABLE IF EXISTS release_dirty`)
+  }
+}

database/migrations/1768620765200_seed_range_priority.ts

@@ -0,0 +1,14 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** Seed ranges of a higher priority are leased first (`seed:plan --priority`), e.g. a gap in recent history. */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('seed_ranges', (table) => {
+      table.smallint('priority').notNullable().defaultTo(0)
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('seed_ranges', (table) => table.dropColumn('priority'))
+  }
+}

database/migrations/1768620765210_binary_part_count.ts

@@ -0,0 +1,21 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Stored parts per binary, kept by IngestService as it inserts parts
+ * (LOG.md 2026-10-06 (7)). claimCompleted counted every binary's parts in
+ * every batch (15% of Postgres' active time); a binary of thousands of
+ * parts that gets a few per batch was counted thousands of times. NULL:
+ * not counted yet (binaries from before), counted exactly on the next new
+ * part. Only parts of binaries without a file are deleted, so it stays exact
+ * until a binary completes.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
+    await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS part_count integer`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS part_count`)
+  }
+}

database/migrations/1768620765220_file_keys.ts

@@ -0,0 +1,69 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)).
+ * file_key is unique per files partition only (D39), so FileStore.insert
+ * looked every key up in every partition: 7 partitions and growing, 35,000
+ * random index reads for a batch of 5,000 files. With file_keys complete it
+ * asks file_keys first, and only keys whose hash is there (a duplicate, or
+ * rarely a collision) are looked up in files.
+ *
+ * New keys come from the files insert statement trigger. The keys from
+ * before are copied in the background (FileKeys.fill, gc:scheduler):
+ * copying them here took over half an hour of HDD reads and held up the
+ * deploy. Until file_keys_fill.complete, FileStore checks every partition.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`)
+    // One row: the fill's position in files.id, the id it must reach, and whether it got there.
+    await q(`
+      CREATE TABLE IF NOT EXISTS file_keys_fill (
+        id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1),
+        next_id bigint NOT NULL DEFAULT 0,
+        until_id bigint,
+        complete boolean NOT NULL DEFAULT false,
+        updated_at timestamptz NOT NULL DEFAULT now())`)
+    await q(`INSERT INTO file_keys_fill (id) VALUES (1) ON CONFLICT DO NOTHING`)
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          WITH added AS (
+            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
+                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
+                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
+            FROM (
+              SELECT release_id, size, name_extension(nm) AS e,
+                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
+              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
+            ) x
+            GROUP BY release_id
+          )
+          UPDATE releases r SET
+            file_count = r.file_count + a.n,
+            size = r.size + a.bytes,
+            extensions = CASE
+              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
+              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
+            search_files = CASE
+              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
+              ELSE nullif(left(concat_ws(' ', r.search_files, (
+                SELECT string_agg(w, ' ') FROM unnest(a.words) w
+                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
+          FROM added a
+          WHERE r.id = a.release_id;
+
+          INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
+          INSERT INTO file_keys (key_hash)
+            SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
+            ON CONFLICT DO NOTHING;
+          RETURN NULL;
+        END $$`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys_fill`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`)
+  }
+}

database/migrations/1768620765230_binaries_awaiting_inspection.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A completed binary whose file is queued for content inspection keeps its
+ * row until inspection has stored the file's fingerprint on it (LOG.md
+ * 2026-10-06 (11)): par2 naming (D16) matches par2 listings against binary
+ * fingerprints and attaches sets to binaries. With the inspection queue
+ * days behind, the GC deleted them first (after 6 hours unfingerprinted,
+ * 48 hours stale). TRUE while waiting; cleared by inspection.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
+    await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS awaiting_inspection boolean`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS awaiting_inspection`)
+  }
+}

database/migrations/1768620765240_binaries_gc_indexes.ts

@@ -0,0 +1,32 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Indexes for the GC's deletes of finished binaries, without the ones
+ * waiting for inspection (LOG.md 2026-10-06 (11)): those can be millions
+ * when inspection is behind, and the deletes must not read past them every
+ * sweep. Built CONCURRENTLY: binaries takes writes all the time.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_done_last_seen_idx ON binaries (last_seen_at)
+       WHERE file_id IS NOT NULL AND awaiting_inspection IS NOT TRUE`
+    )
+    await this.db.rawQuery(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_awaiting_inspection_idx ON binaries (last_seen_at)
+       WHERE awaiting_inspection IS TRUE`
+    )
+    await this.db.rawQuery(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_open_last_seen_idx ON binaries (last_seen_at)
+       WHERE completed_at IS NULL OR conflicted_at IS NOT NULL`
+    )
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_open_last_seen_idx`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_awaiting_inspection_idx`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_done_last_seen_idx`)
+  }
+}

database/migrations/1768620765250_release_completeness_narrower.ts

@@ -0,0 +1,94 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Release completeness (D24) for fewer releases, and without binaries
+ * (LOG.md 2026-10-07 (7)).
+ *
+ * - Only releases whose completeness can change are noted for the sweep: a
+ *   par2 listing, a file_total, or complete now (it can become incomplete).
+ *   For any other release release_complete() is false by definition; noting
+ *   every release a file went into made a sweep check 430,000 releases in
+ *   51 minutes.
+ * - release_complete() matches listed files by name against the files
+ *   themselves (coalesce(real_name, filename)); the size match still goes
+ *   through a binary when there is one. Seeding no longer creates binaries
+ *   for most files and the GC deletes them, so a name only reachable through
+ *   a binary was lost.
+ * - Notes that don't qualify are dropped.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`
+      CREATE OR REPLACE FUNCTION release_completeness_matters(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
+        SELECT EXISTS (SELECT 1 FROM releases r WHERE r.id = rid AND (r.file_total IS NOT NULL OR r.complete))
+            OR EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid)
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL AND release_completeness_matters(rid)
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
+        WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
+        -- The release's files, once, by index (read only when there's a par2 listing to check).
+        have AS MATERIALIZED (
+          SELECT b.size, coalesce(f.real_name, f.filename) AS name
+          FROM files f LEFT JOIN binaries b ON b.file_id = f.id
+          WHERE f.release_id = rid)
+        SELECT coalesce((
+          SELECT CASE
+            WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
+            WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
+              SELECT 1 FROM release_files rf
+              WHERE rf.release_id = rid AND rf.size >= 65536
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.name = rf.name))
+            ELSE false END
+          FROM r), false)
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          WITH added AS (
+            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
+                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
+                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
+            FROM (
+              SELECT release_id, size, name_extension(nm) AS e,
+                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
+              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
+            ) x
+            GROUP BY release_id
+          )
+          UPDATE releases r SET
+            file_count = r.file_count + a.n,
+            size = r.size + a.bytes,
+            extensions = CASE
+              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
+              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
+            search_files = CASE
+              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
+              ELSE nullif(left(concat_ws(' ', r.search_files, (
+                SELECT string_agg(w, ' ') FROM unnest(a.words) w
+                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
+          FROM added a
+          WHERE r.id = a.release_id;
+
+          -- Only releases whose completeness can change (LOG.md 2026-10-07 (7)).
+          INSERT INTO release_dirty (release_id)
+            SELECT n.release_id FROM (SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL) n
+            WHERE release_completeness_matters(n.release_id);
+          INSERT INTO file_keys (key_hash)
+            SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
+            ON CONFLICT DO NOTHING;
+          RETURN NULL;
+        END $$`)
+    await q(`DELETE FROM release_dirty d WHERE NOT release_completeness_matters(d.release_id)`)
+  }
+
+  async down() {
+    // The previous definitions are in migrations …180, …190 and …220.
+    throw new Error('Not reversible: re-run the earlier definitions by hand if needed.')
+  }
+}

database/migrations/1768620765260_inspection_queue.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The content inspection backlog (LOG.md 2026-10-07 (8)): files waiting to be
+ * inspected, in Postgres instead of as BullMQ jobs. 5.4M waiting jobs took
+ * 3.3 GB of Redis memory and most of its single thread; InspectionQueue.feed
+ * keeps at most INSPECTION_FEED_MAX of them in BullMQ, where the workers and
+ * nodes take them. Lower priority first, then oldest.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS inspection_queue (
+        file_id bigint PRIMARY KEY,
+        priority smallint NOT NULL DEFAULT 0,
+        queued_at timestamptz NOT NULL DEFAULT now())`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS inspection_queue_order_idx ON inspection_queue (priority, queued_at)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS inspection_queue`)
+  }
+}

database/migrations/1768620765270_garbage_filter.ts

@@ -0,0 +1,64 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Dropping garbage uploads early (CONCEPT.md D42): random-named uploads that
+ * never get a real name or password.
+ * - upload_signatures: per posting tool (from the Message-ID) and first group,
+ *   how many probed random-named releases got a real name; `dropped` ones
+ *   aren't stored any more (GarbageFilter).
+ * - signature_probes: a sample of probed random-named releases, what
+ *   upload_signatures is learned from (the outcome is read from the release
+ *   at learning time, so names that come later count too).
+ * - dropped_releases: releases deleted after their probe, by release_key, so
+ *   seeding and ingest don't add them again.
+ * - is_real_release_name(): a name someone would search for. is_readable_name
+ *   lets spam through (usenetszene net, VTS_01_1.VOB, Nieuwe map, 100 MiB):
+ *   this also wants a year, episode, resolution, codec or language tag, or
+ *   three words (LOG.md 2026-10-08 (5)). No `?` in its regexes: knex takes it
+ *   for a binding (hence {0,1}).
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS upload_signatures (
+        tool varchar(32) NOT NULL,
+        group_name text NOT NULL,
+        probed integer NOT NULL DEFAULT 0,
+        named integer NOT NULL DEFAULT 0,
+        dropped boolean NOT NULL DEFAULT false,
+        updated_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (tool, group_name))`)
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS signature_probes (
+        release_id bigint PRIMARY KEY,
+        tool varchar(32) NOT NULL,
+        group_name text NOT NULL,
+        probed_at timestamptz NOT NULL DEFAULT now())`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS signature_probes_probed_at_idx ON signature_probes (probed_at)`)
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS dropped_releases (
+        release_key varchar(40) PRIMARY KEY,
+        reason varchar(16) NOT NULL,
+        dropped_at timestamptz NOT NULL DEFAULT now())`)
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION is_real_release_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+        SELECT name IS NOT NULL AND is_readable_name(name)
+          AND NOT (
+            name ~* '(usenet|video_ts|nieuwe map|new folder|(^|[^a-z0-9])mib([^a-z0-9]|$))'
+            OR name ~* '^(vts_|sample|cd[0-9])'
+            OR name ~ '^\\(.*\\)$'
+            OR name ~ '^[A-Za-z0-9]{8,}$')
+          AND (
+            name ~* '(^|[^a-z0-9])((19|20)[0-9]{2}|(480|576|720|1080|2160)[pi]|dl|nl)([^a-z0-9]|$)'
+            OR name ~* '(s[0-9]{2}e[0-9]{2}|x26[45]|h\\.{0,1}26[45]|hevc|web[- .]{0,1}(dl|rip)|bluray|remux|german|dutch|multi|flac|mp3|epub|pdf|ac3|dts|xxx)'
+            OR (SELECT count(*) FROM regexp_matches(name, '[A-Za-z]{3,}', 'g')) >= 3)
+      $$`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP FUNCTION IF EXISTS is_real_release_name(text)`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS dropped_releases`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS signature_probes`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS upload_signatures`)
+  }
+}

database/migrations/1768620765280_spot_listing_reads.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Listed spots (D29) were only read from their overview: no description, no
+ * NZB, so neither a password in the spot's text nor the NZB's own
+ * `<meta type="password">` was known (LOG.md 2026-10-08 (10)). nzb_read_at:
+ * when the spot's article and NZB were read for that (SpotService.readListing),
+ * also set while a reader holds it. The index finds the ones still to read,
+ * newest first.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS nzb_read_at timestamptz`)
+    await this.db.rawQuery(
+      `CREATE INDEX IF NOT EXISTS spots_listing_unread_idx ON spots (posted_at DESC) WHERE status = 'listed' AND nzb_read_at IS NULL`
+    )
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX IF EXISTS spots_listing_unread_idx`)
+    await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS nzb_read_at`)
+  }
+}

database/migrations/1768620765290_file_details.ts

@@ -0,0 +1,28 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * What inspection learns about a file (content type, archive listing) lives
+ * in file_details, on the NVMe, instead of in `files` (LOG.md 2026-10-08 (14)).
+ * `files` sits on the HDD and was updated after its insert by every probe:
+ * a new row version on a random page and an entry in each of its indexes
+ * (3.7M updates against 49.5M inserts in a day, 27% HOT). Its rows are now
+ * only inserted (and moved between releases or renamed, rarely).
+ * The existing values are copied by `files:move-details` after this deploy;
+ * a later migration drops the columns.
+ */
+export default class extends BaseSchema {
+  async up() {
+    // The foreign key locks `files` briefly: take the lock first, so it can't deadlock with writers.
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '60s'`)
+    await this.db.rawQuery(`LOCK TABLE files IN SHARE ROW EXCLUSIVE MODE`)
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS file_details (
+        file_id bigint PRIMARY KEY REFERENCES files (id) ON DELETE CASCADE,
+        content_type varchar(16),
+        archive jsonb)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS file_details`)
+  }
+}

database/schema.ts

@@ -33,14 +33,16 @@ export class AuthAccessTokenSchema extends BaseModel {
 }
 
 export class BinarySchema extends BaseModel {
-  static $columns = ['completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
+  static $columns = ['awaitingInspection', 'completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'partCount', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
   $columns = BinarySchema.$columns
+  @column()
+  declare awaitingInspection: boolean | null
   @column.dateTime()
   declare completedAt: DateTime | null
   @column.dateTime()
   declare conflictedAt: DateTime | null
   @column()
-  declare fileId: number | null
+  declare fileId: bigint | number | null
   @column()
   declare fileKey: string
   @column.dateTime()
@@ -57,6 +59,8 @@ export class BinarySchema extends BaseModel {
   declare name: string
   @column()
   declare parentBinaryId: bigint | number | null
+  @column()
+  declare partCount: number | null
   @column.dateTime()
   declare postedAt: DateTime | null
   @column()
@@ -147,6 +151,17 @@ export class DeferredRangeSchema extends BaseModel {
   declare lastArticle: bigint | number
 }
 
+export class DroppedReleaseSchema extends BaseModel {
+  static $columns = ['droppedAt', 'reason', 'releaseKey'] as const
+  $columns = DroppedReleaseSchema.$columns
+  @column.dateTime()
+  declare droppedAt: DateTime
+  @column()
+  declare reason: string
+  @column({ isPrimary: true })
+  declare releaseKey: string
+}
+
 export class FetchRangeSchema extends BaseModel {
   static $columns = ['backfill', 'createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
   $columns = FetchRangeSchema.$columns
@@ -172,6 +187,39 @@ export class FetchRangeSchema extends BaseModel {
   declare updatedAt: DateTime
 }
 
+export class FileDetailSchema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'fileId'] as const
+  $columns = FileDetailSchema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column({ isPrimary: true })
+  declare fileId: bigint | number
+}
+
+export class FileKeySchema extends BaseModel {
+  static $columns = ['keyHash'] as const
+  $columns = FileKeySchema.$columns
+  @column({ isPrimary: true })
+  declare keyHash: bigint | number
+}
+
+export class FileKeysFillSchema extends BaseModel {
+  static $columns = ['complete', 'id', 'nextId', 'untilId', 'updatedAt'] as const
+  $columns = FileKeysFillSchema.$columns
+  @column()
+  declare complete: boolean
+  @column({ isPrimary: true })
+  declare id: number
+  @column()
+  declare nextId: bigint | number
+  @column()
+  declare untilId: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime
+}
+
 export class FileSchema extends BaseModel {
   static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
   $columns = FileSchema.$columns
@@ -192,7 +240,7 @@ export class FileSchema extends BaseModel {
   @column()
   declare groups: any | null
   @column({ isPrimary: true })
-  declare id: number
+  declare id: bigint | number
   @column()
   declare parts: number
   @column()
@@ -229,7 +277,7 @@ export class FilesP0Schema extends BaseModel {
   @column()
   declare groups: any | null
   @column({ isPrimary: true })
-  declare id: number
+  declare id: bigint | number
   @column()
   declare parts: number
   @column()
@@ -266,7 +314,7 @@ export class FilesP1000000Schema extends BaseModel {
   @column()
   declare groups: any | null
   @column({ isPrimary: true })
-  declare id: number
+  declare id: bigint | number
   @column()
   declare parts: number
   @column()
@@ -303,7 +351,7 @@ export class FilesP5000000Schema extends BaseModel {
   @column()
   declare groups: any | null
   @column({ isPrimary: true })
-  declare id: number
+  declare id: bigint | number
   @column()
   declare parts: number
   @column()
@@ -345,6 +393,17 @@ export class GroupSchema extends BaseModel {
   declare updatedAt: DateTime | null
 }
 
+export class InspectionQueueSchema extends BaseModel {
+  static $columns = ['fileId', 'priority', 'queuedAt'] as const
+  $columns = InspectionQueueSchema.$columns
+  @column({ isPrimary: true })
+  declare fileId: bigint | number
+  @column()
+  declare priority: number
+  @column.dateTime()
+  declare queuedAt: DateTime
+}
+
 export class PartConflictSchema extends BaseModel {
   static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const
   $columns = PartConflictSchema.$columns
@@ -391,6 +450,13 @@ export class PartSchema extends BaseModel {
   declare pcrc32: string | null
 }
 
+export class ReleaseDirtySchema extends BaseModel {
+  static $columns = ['releaseId'] as const
+  $columns = ReleaseDirtySchema.$columns
+  @column()
+  declare releaseId: bigint | number
+}
+
 export class ReleaseFileSchema extends BaseModel {
   static $columns = ['md5', 'md516K', 'name', 'par2FileId', 'releaseId', 'setId', 'size'] as const
   $columns = ReleaseFileSchema.$columns
@@ -411,7 +477,7 @@ export class ReleaseFileSchema extends BaseModel {
 }
 
 export class ReleaseSchema extends BaseModel {
-  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'extensions', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const
+  static $columns = ['category', 'complete', 'container', 'contents', 'contentsWalkedAt', 'encrypted', 'extensions', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const
   $columns = ReleaseSchema.$columns
   @column()
   declare category: number | null
@@ -421,6 +487,8 @@ export class ReleaseSchema extends BaseModel {
   declare container: string | null
   @column()
   declare contents: any | null
+  @column.dateTime()
+  declare contentsWalkedAt: DateTime | null
   @column()
   declare encrypted: boolean | null
   @column()
@@ -477,7 +545,7 @@ export class SeedNodeSchema extends BaseModel {
 }
 
 export class SeedRangeSchema extends BaseModel {
-  static $columns = ['attempts', 'createdAt', 'doneAt', 'endId', 'error', 'files', 'fragments', 'groupName', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'startId', 'status'] as const
+  static $columns = ['attempts', 'createdAt', 'doneAt', 'endId', 'error', 'files', 'fragments', 'groupName', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'startId', 'status'] as const
   $columns = SeedRangeSchema.$columns
   @column()
   declare attempts: number
@@ -508,6 +576,8 @@ export class SeedRangeSchema extends BaseModel {
   @column()
   declare nodeId: number | null
   @column()
+  declare priority: number
+  @column()
   declare startId: bigint | number
   @column()
   declare status: string
@@ -528,8 +598,21 @@ export class SfvEntrySchema extends BaseModel {
   declare releaseId: bigint | number
 }
 
+export class SignatureProbeSchema extends BaseModel {
+  static $columns = ['groupName', 'probedAt', 'releaseId', 'tool'] as const
+  $columns = SignatureProbeSchema.$columns
+  @column()
+  declare groupName: string
+  @column.dateTime()
+  declare probedAt: DateTime
+  @column({ isPrimary: true })
+  declare releaseId: bigint | number
+  @column()
+  declare tool: string
+}
+
 export class SpotSchema extends BaseModel {
-  static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const
+  static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'nzbReadAt', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const
   $columns = SpotSchema.$columns
   @column()
   declare articleNumber: bigint | number | null
@@ -552,6 +635,8 @@ export class SpotSchema extends BaseModel {
   @column({ isPrimary: true })
   declare messageId: string
   @column.dateTime()
+  declare nzbReadAt: DateTime | null
+  @column.dateTime()
   declare postedAt: DateTime | null
   @column()
   declare poster: string
@@ -590,6 +675,23 @@ export class StagedSegmentSchema extends BaseModel {
   declare stagedAt: DateTime
 }
 
+export class UploadSignatureSchema extends BaseModel {
+  static $columns = ['dropped', 'groupName', 'named', 'probed', 'tool', 'updatedAt'] as const
+  $columns = UploadSignatureSchema.$columns
+  @column()
+  declare dropped: boolean
+  @column()
+  declare groupName: string
+  @column()
+  declare named: number
+  @column()
+  declare probed: number
+  @column({ isPrimary: true })
+  declare tool: string
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime
+}
+
 export class UserSchema extends BaseModel {
   static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
   $columns = UserSchema.$columns

lib/content/archive_names.ts

@@ -23,3 +23,18 @@ export function isLaterVolume(name: string): boolean {
   if (part) return Number(part[1]) > 1
   return /\.r\d{2,3}$/i.test(name) || /\.vol\d+[+-]\d+\.par2$/i.test(name)
 }
+
+/**
+ * The set and 1-based number of a RAR volume: `x.part03.rar` is volume 3 of
+ * `x`, and in the old naming `x.rar` is 1, `x.r00` 2, `x.r01` 3. Null for
+ * other names.
+ */
+export function rarVolume(name: string): { base: string; number: number } | null {
+  const part = /^(.*)\.part(\d+)\.rar$/i.exec(name)
+  if (part) return { base: part[1].toLowerCase(), number: Number(part[2]) }
+  const old = /^(.*)\.r(\d{2,3})$/i.exec(name)
+  if (old) return { base: old[1].toLowerCase(), number: Number(old[2]) + 2 }
+  const first = /^(.*)\.rar$/i.exec(name)
+  if (first) return { base: first[1].toLowerCase(), number: 1 }
+  return null
+}

lib/content/index.ts

@@ -2,6 +2,6 @@
  * Content recognition by magic bytes, independent of AdonisJS.
  */
 export { sniffContentType, type ContentType } from './sniff.js'
-export { isArchiveStart, isLaterVolume } from './archive_names.js'
+export { isArchiveStart, isLaterVolume, rarVolume } from './archive_names.js'
 export { parseSfv, sfvReleaseName, type SfvEntry } from './sfv.js'
 export { NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, nicovideoReleaseName, zstdPrefix, tarEntries, type NicovideoInfo } from './nicovideo.js'

lib/content/probe.ts

@@ -0,0 +1,205 @@
+import { createHash } from 'node:crypto'
+import { decodeSegment, YencCrcError, YencError, YencFile, type DecodedSegment } from '../yenc/index.js'
+import { isPar2, parsePar2, type Par2Set } from '../par2/index.js'
+import { parseRarHeaders, type RarEntry, type RarInfo } from '../rar/parse.js'
+import { extractEntry } from '../rar/extract.js'
+import { walkRar } from '../rar/walk.js'
+import { SegmentReader, type BodyFetcher } from '../rar/segment_reader.js'
+import { listSevenZip } from '../sevenzip/list.js'
+import { sniffContentType, type ContentType } from './sniff.js'
+import { NICOVIDEO_FILE, nicovideoInfo, zstdPrefix, type NicovideoInfo } from './nicovideo.js'
+
+/**
+ * Content inspection's download-and-parse half (CONCEPT.md D21, D41),
+ * without a database: what the indexer needs to know about a file, from its
+ * segments. The indexer prepares a task from the database, the probe runs
+ * here or on a node (articles through the seed API's relay), and the
+ * indexer applies the result. Tasks and results are plain JSON.
+ */
+
+export interface TaskSegment {
+  number: number
+  id: string
+  /** Article size in bytes. */
+  size?: number
+}
+
+/** Inspect one file: its first segment, and more where its kind needs it. */
+export interface InspectTask {
+  kind: 'inspect'
+  fileId: number
+  /** real_name, else filename. */
+  name: string
+  segments: TaskSegment[]
+  /** For a 7z listing: the archive's volume count and its last volume's segments; null when volumes are missing. */
+  sevenZip: { volumes: number; last: TaskSegment[] } | null
+}
+
+export interface InspectProbe {
+  /** The whole file's size, from the yEnc header. */
+  size: number
+  contentType: ContentType | null
+  md5_16k: string | null
+  /** The file name in the yEnc header. */
+  yencName: string
+  nicovideo?: NicovideoInfo | null
+  /** A whole small SFV (latin1) or NZB (utf8). */
+  text?: string | null
+  par2?: Par2Set | null
+  archive?: RarInfo | null
+  /** NZBs read from inside an unsplit archive, and how many NZB entries it lists. */
+  archiveNzbs?: { entryName: string; xml: string }[]
+  archiveNzbEntries?: number
+}
+
+/** Walk a RAR set's headers for its complete listing (D40). */
+export interface WalkTask {
+  kind: 'walk'
+  releaseId: number
+  volumes: TaskSegment[][]
+  maxSegments: number
+}
+
+export interface WalkProbe {
+  entries: Omit<RarEntry, 'stored' | 'dataOffset'>[]
+  complete: boolean
+  stopped?: string
+  headersEncrypted: boolean
+  segments: number
+}
+
+export type ContentTask = InspectTask | WalkTask
+
+/** Small posted files (NZBs, SFVs) that are read completely. */
+const WHOLE_MAX_PARTS = 40
+const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+/** Par2 index files are small; recovery volumes can be large and are not needed. */
+const PAR2_MAX_PARTS = 20
+const FIRST_CHUNK = 16 * 1024
+
+/** MD5 of the first 16 KiB of a file, as par2 records it; undefined when the data is shorter than that but the file isn't. */
+export function chunkMd5(data: Buffer, fileSize: number): string | undefined {
+  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
+  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
+}
+
+/** A "no such article" answer: 430 from the news server, 410 from the seed relay. */
+export function isMissingArticle(error: unknown): boolean {
+  return /(^|[^0-9])(430|410)([^0-9]|$)/.test(String((error as any)?.message ?? error)) && !/timed out|no answer/i.test(String((error as any)?.message))
+}
+
+/** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */
+export function decodeBody(body: Buffer): DecodedSegment {
+  try {
+    return decodeSegment(body)
+  } catch (error) {
+    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
+    return decodeSegment(body, { lenientSize: true })
+  }
+}
+
+/**
+ * All of a (small) file whose first segment is already decoded; null when it is too big.
+ * The first segment isn't downloaded again (it was, for every par2 set, NZB and SFV).
+ */
+export async function readWhole(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Buffer | null> {
+  const bytes = segments.reduce((sum, p) => sum + (p.size ?? 0), 0)
+  if (segments.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
+  if (segments.length === 1) return first.data
+  const assembled = YencFile.fromSegment(first)
+  assembled.add(first)
+  for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id), { lenientSize: true }))
+  return assembled.getBuffer()
+}
+
+/** A par2 file whose first segment is already decoded, with its other parts if any; null when it is not par2. */
+export async function readPar2(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Par2Set | null> {
+  if (!isPar2(first.data)) return null
+  if (segments.length > PAR2_MAX_PARTS) return null
+  if (segments.length === 1) return parsePar2(first.data)
+  const assembled = YencFile.fromSegment(first)
+  assembled.add(first)
+  for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id)))
+  return parsePar2(assembled.getBuffer())
+}
+
+export function isNzbEntry(entry: Pick<RarEntry, 'name' | 'encrypted' | 'directory'>): boolean {
+  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+}
+
+/** NZBs that can be read from a single, unencrypted archive. */
+function hasNzbEntries(info: RarInfo): boolean {
+  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+}
+
+export async function probeInspection(
+  task: InspectTask,
+  fetch: BodyFetcher,
+  warn: (message: string) => void = () => {}
+): Promise<InspectProbe> {
+  let body: Buffer
+  try {
+    body = await fetch(task.segments[0].id)
+  } catch (error: any) {
+    // The server no longer has it (430; 410 through the seed relay), typically a takedown. A final
+    // answer, and an expensive one: Eweka takes ~450 ms per missing article, so it isn't retried
+    // (LOG.md 2026-10-06 (14)).
+    if (isMissingArticle(error)) return { size: 0, contentType: 'missing', md5_16k: null, yencName: '' }
+    throw error
+  }
+  const segment = decodeBody(body)
+  const size = segment.meta.begin.size
+  const contentType = sniffContentType(segment.data)
+  const probe: InspectProbe = {
+    size,
+    contentType,
+    md5_16k: chunkMd5(segment.data, size) ?? null,
+    yencName: segment.meta.begin.name,
+  }
+  // An encrypted NZB, rclone crypt or yEncBin's container: nothing to read.
+  if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') return probe
+  if (NICOVIDEO_FILE.test(task.name)) {
+    probe.nicovideo = nicovideoInfo(await zstdPrefix(segment.data))
+    return probe
+  }
+  if (/\.sfv$/i.test(task.name)) {
+    probe.text = (await readWhole(segment, task.segments, fetch))?.toString('latin1') ?? null
+    return probe
+  }
+  if (contentType === 'nzb') {
+    probe.text = (await readWhole(segment, task.segments, fetch))?.toString('utf8') ?? null
+    return probe
+  }
+  probe.par2 = await readPar2(segment, task.segments, fetch)
+  if (probe.par2) return probe
+  if (contentType === '7z') {
+    probe.archive = task.sevenZip ? await listSevenZip(segment.data, task.sevenZip.volumes, task.sevenZip.last, fetch, warn) : null
+    return probe
+  }
+  const info = parseRarHeaders(segment.data)
+  probe.archive = info
+  if (info && hasNzbEntries(info)) {
+    probe.archiveNzbs = []
+    const archive = await readWhole(segment, task.segments, fetch)
+    const archived = archive && parseRarHeaders(archive)
+    const entries = archived ? archived.entries.filter(isNzbEntry) : []
+    probe.archiveNzbEntries = entries.length
+    for (const entry of entries) {
+      const xml = await extractEntry(archive!, entry, undefined, warn)
+      if (xml) probe.archiveNzbs.push({ entryName: entry.name, xml: xml.toString('utf8') })
+    }
+  }
+  return probe
+}
+
+export async function probeWalk(task: WalkTask, fetch: BodyFetcher): Promise<WalkProbe> {
+  const reader = new SegmentReader(task.volumes, task.maxSegments, fetch)
+  const walked = await walkRar(reader)
+  return {
+    entries: walked.entries.map(({ stored: _s, dataOffset: _d, ...entry }) => entry),
+    complete: walked.complete,
+    stopped: walked.stopped,
+    headersEncrypted: walked.headersEncrypted,
+    segments: reader.fetched,
+  }
+}

lib/content/sniff.ts

@@ -6,6 +6,8 @@
  * (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy.
  */
 export type ContentType =
+  /** Not sniffed: the server no longer has the file's first article (inspection records it, no retries). */
+  | 'missing'
   | 'nzbe'
   | 'par2'
   | 'rar4'

lib/garbage/index.ts

@@ -0,0 +1,52 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * Upload signatures (CONCEPT.md D42): which tool posted an article, read from
+ * its Message-ID, and the group it was posted to first. Together they predict
+ * whether a random-named upload will ever get a name: ngPost uploads in
+ * a.b.ath almost never do, nyuu uploads in a.b.friends always do (LOG.md
+ * 2026-10-08 (5)).
+ */
+const TOOLS: [string, RegExp][] = [
+  // FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu: 12 random capital+small letter pairs, a ms timestamp
+  ['nyuu', /^(?:[A-Z][a-z]){12}-\d{13}@nyuu$/],
+  ['nyuu-host', /^(?:[A-Z][a-z]){12}-\d{13}@.+$/],
+  // b94268bc702040f7b811d5318f727e4f@ngPost: a UUID's hex
+  ['ngPost', /^[0-9a-f]{32}@ngPost$/],
+  ['yuun', /^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$/],
+  ['hex32', /^[0-9a-f]{32}@[A-Za-z0-9.-]+$/],
+  ['JBinUp', /@JBin(?:Up|Down)\.local$/],
+  // part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local
+  ['partNofM', /^part\d+of\d+\./i],
+  // 1790923856877$1729905757859$1@localhost
+  ['ts-dollar', /^\d{13}\$\d+\$\d+@localhost$/],
+  // t3Y4mV92YCp0McxDc5@zLZlsw.RYqes: random local part and random host
+  ['random', /^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$/],
+]
+
+/** The posting tool of a Message-ID (without angle brackets), or 'other'. */
+export function uploadTool(messageId: string | null | undefined): string {
+  if (!messageId) return 'other'
+  for (const [tool, pattern] of TOOLS) if (pattern.test(messageId)) return tool
+  return 'other'
+}
+
+/** The group of a signature: the first the upload was posted to, as stored in `groups`. */
+export function signatureGroup(groups: unknown): string {
+  return Array.isArray(groups) && typeof groups[0] === 'string' ? groups[0] : ''
+}
+
+/** The key of a signature in a Set. */
+export function signatureKey(tool: string, group: string) {
+  return `${tool}\t${group}`
+}
+
+/**
+ * Whether an upload is in the share of dropped signatures that is still kept
+ * (and probed), so their rates keep being measured. Decided per release key,
+ * so a release is kept or dropped as a whole.
+ */
+export function explored(key: string, percent: number): boolean {
+  if (percent <= 0) return false
+  return createHash('md5').update(key).digest().readUInt32BE(0) % 10_000 < percent * 100
+}

lib/nntp/xover_client.ts

@@ -1,6 +1,9 @@
+import { promisify } from 'node:util'
 import net from 'node:net'
 import tls from 'node:tls'
-import { deflateSync, unzipSync } from 'node:zlib'
+import { deflate, deflateSync, unzipSync } from 'node:zlib'
+
+const deflateAsync = promisify(deflate)
 
 export interface XoverClientOptions {
   host: string
@@ -10,7 +13,7 @@ export interface XoverClientOptions {
   secure?: boolean
   /** Hostname the TLS certificate must match, when connecting through a tunnel (default: host). */
   servername?: string
-  /** A command whose reply stalls this long fails, and the connection with it. */
+  /** A reply with no data for this long fails, and the connection with it (inactivity, not duration). */
   timeoutMs?: number
   /** Ask for gzip-compressed overview data (XFEATURE COMPRESS GZIP), when the server has it. */
   compress?: boolean
@@ -100,7 +103,9 @@ export class XoverClient {
       const reply = await this.command(`XOVER ${from}-${to}`)
       if (reply.startsWith('420') || reply.startsWith('423')) return deflateSync(Buffer.alloc(0))
       await this.expect(reply, ['224'])
-      return reply.includes('COMPRESS=GZIP') ? (await this.compressedRaw()).raw : deflateSync(Buffer.from(await this.plainBody()))
+      if (reply.includes('COMPRESS=GZIP')) return (await this.compressedRaw()).raw
+      // Plain replies (the relay's default): compressed off the event loop, at the fastest level.
+      return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 })
     })
   }
 

lib/rar/extract.ts

@@ -0,0 +1,48 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { storedEntryData, type RarEntry } from './parse.js'
+
+const run = promisify(execFile)
+
+/** Tools that print one archive entry to stdout, tried in order. */
+const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
+  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
+  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
+]
+
+/**
+ * The unpacked data of one entry of a complete (single-volume) archive:
+ * copied out when stored, else unpacked with bsdtar (libarchive, in the
+ * Docker image) or unrar. Null when that is not possible.
+ */
+export async function extractEntry(
+  archive: Buffer,
+  entry: RarEntry,
+  maxBytes = 64 * 1024 * 1024,
+  warn: (message: string) => void = () => {}
+): Promise<Buffer | null> {
+  const stored = storedEntryData(archive, entry)
+  if (stored) return stored
+  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
+  if (entry.size !== null && entry.size > maxBytes) return null
+
+  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
+  try {
+    const file = path.join(dir, 'archive.rar')
+    await writeFile(file, archive)
+    for (const [tool, args] of EXTRACTORS) {
+      try {
+        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
+        if (stdout.length > 0) return stdout
+      } catch (error: any) {
+        if (error.code !== 'ENOENT') warn(`${tool} could not extract "${entry.name}": ${error.message}`)
+      }
+    }
+    return null
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}

lib/rar/index.ts

@@ -3,3 +3,6 @@
  * contents from the first segment of a first volume. See CONCEPT.md (D19).
  */
 export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'
+export { walkRar, type VolumeReader, type RarWalk } from './walk.js'
+export { SegmentReader, type BodyFetcher } from './segment_reader.js'
+export { extractEntry } from './extract.js'

lib/rar/parse.ts

@@ -31,6 +31,12 @@ export interface RarInfo {
   entries: RarEntry[]
   /** The buffer ended before the end-of-archive header (normal for one segment). */
   truncated: boolean
+  /**
+   * Offset of the first block that was not read: where the next header
+   * starts (possibly beyond the buffer, after an entry's data), or the
+   * header the buffer cut off. After the end-of-archive header, its end.
+   */
+  next: number
 }
 
 const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
@@ -45,11 +51,16 @@ export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
 /**
  * Lists a RAR archive from its headers only, so the first segment of a first
  * volume is enough. Returns null when the data is not a RAR archive.
+ *
+ * With `continueAs`, `data` starts at a header in the middle of a volume (no
+ * signature), e.g. where a previous parse said the next header is.
  */
-export function parseRarHeaders(data: Buffer): RarInfo | null {
+export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null {
+  if (continueAs === 'rar4') return parseRar4(data, 0)
+  if (continueAs === 'rar5') return parseRar5(data, 0)
   const format = rarFormat(data)
-  if (format === 'rar4') return parseRar4(data)
-  if (format === 'rar5') return parseRar5(data)
+  if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length)
+  if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length)
   return null
 }
 
@@ -60,7 +71,7 @@ const R4_FILE = 0x74
 const R4_END = 0x7b
 const R4_LONG_BLOCK = 0x8000
 
-function parseRar4(data: Buffer): RarInfo {
+function parseRar4(data: Buffer, start: number): RarInfo {
   const info: RarInfo = {
     format: 'rar4',
     volume: false,
@@ -69,8 +80,9 @@ function parseRar4(data: Buffer): RarInfo {
     headersEncrypted: false,
     entries: [],
     truncated: true,
+    next: start,
   }
-  let p = RAR4_MARKER.length
+  let p = start
 
   while (p + 7 <= data.length) {
     const headCrc = data.readUInt16LE(p)
@@ -81,6 +93,7 @@ function parseRar4(data: Buffer): RarInfo {
 
     if (type === R4_END) {
       info.truncated = false
+      p += size
       break
     }
 
@@ -130,6 +143,7 @@ function parseRar4(data: Buffer): RarInfo {
   if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
     info.firstVolume = true
   }
+  info.next = p
   return info
 }
 
@@ -205,7 +219,7 @@ function vint(data: Buffer, offset: number): { value: number; next: number } | n
   return null
 }
 
-function parseRar5(data: Buffer): RarInfo {
+function parseRar5(data: Buffer, start: number): RarInfo {
   const info: RarInfo = {
     format: 'rar5',
     volume: false,
@@ -214,8 +228,9 @@ function parseRar5(data: Buffer): RarInfo {
     headersEncrypted: false,
     entries: [],
     truncated: true,
+    next: start,
   }
-  let p = RAR5_MARKER.length
+  let p = start
 
   while (p + 5 <= data.length) {
     const headCrc = data.readUInt32LE(p)
@@ -246,6 +261,7 @@ function parseRar5(data: Buffer): RarInfo {
 
     if (type.value === R5_END) {
       info.truncated = false
+      p = headerEnd
       break
     }
     if (type.value === R5_ENCRYPTION) {
@@ -270,6 +286,7 @@ function parseRar5(data: Buffer): RarInfo {
     if (next <= p) break
     p = next
   }
+  info.next = p
   return info
 }
 

lib/rar/segment_reader.ts

@@ -0,0 +1,82 @@
+import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '../yenc/index.js'
+import type { VolumeReader } from './walk.js'
+
+/** Downloads one article body (on a pooled connection, or through the seed API's relay). */
+export type BodyFetcher = (messageId: string) => Promise<Buffer>
+
+/**
+ * Reads volumes (each a list of segment message ids, in order) a segment at
+ * a time, finding the segment for an offset from the part size (yEnc
+ * posters use one size per set) and correcting from the `=ypart` offsets
+ * when a guess is off. Stops reading after `maxSegments` downloads.
+ */
+export class SegmentReader implements VolumeReader {
+  fetched = 0
+  private partSize = 0
+  private readonly sizes = new Map<number, number>()
+  private readonly cache = new Map<string, DecodedSegment>()
+
+  constructor(
+    private readonly files: { id: string }[][],
+    private readonly maxSegments: number,
+    private readonly fetchBody: BodyFetcher
+  ) {}
+
+  get volumes() {
+    return this.files.length
+  }
+
+  size(v: number) {
+    return this.sizes.get(v) ?? null
+  }
+
+  async read(v: number, offset: number, more: boolean): Promise<Buffer | null> {
+    const count = this.files[v].length
+    let index = this.partSize ? Math.min(count - 1, Math.floor(offset / this.partSize)) : 0
+    for (let tries = 0; tries < 4; tries++) {
+      const segment = await this.segment(v, index)
+      if (!segment) return null
+      const end = segment.offset + segment.data.length
+      if (offset >= segment.offset && offset < end) {
+        const data = segment.data.subarray(offset - segment.offset)
+        if (!more || index + 1 >= count) return data
+        const next = await this.segment(v, index + 1)
+        return next ? Buffer.concat([data, next.data]) : null
+      }
+      const step = Math.floor((offset - segment.offset) / Math.max(1, segment.data.length))
+      const moved = Math.max(0, Math.min(count - 1, index + (step === 0 ? (offset < segment.offset ? -1 : 1) : step)))
+      if (moved === index) return null
+      index = moved
+    }
+    return null
+  }
+
+  private async segment(v: number, index: number): Promise<DecodedSegment | null> {
+    const key = `${v}:${index}`
+    const cached = this.cache.get(key)
+    if (cached) return cached
+    const part = this.files[v][index]
+    if (!part?.id || this.fetched >= this.maxSegments) return null
+    this.fetched++
+    let body: Buffer
+    try {
+      body = await this.fetchBody(part.id)
+    } catch {
+      // A missing article (or the relay failing) ends the walk with what was found.
+      return null
+    }
+    let segment: DecodedSegment
+    try {
+      segment = decodeSegment(body)
+    } catch (error) {
+      if (!(error instanceof YencError) || error instanceof YencCrcError) return null
+      segment = decodeSegment(body, { lenientSize: true })
+    }
+    this.sizes.set(v, segment.meta.begin.size)
+    if (!this.partSize && segment.meta.part) this.partSize = segment.meta.part.end - segment.meta.part.begin + 1
+    // Walks move forward: the last few segments are enough.
+    if (this.cache.size >= 4) this.cache.delete(this.cache.keys().next().value!)
+    this.cache.set(key, segment)
+    return segment
+  }
+}

lib/rar/walk.ts

@@ -0,0 +1,77 @@
+import { parseRarHeaders, rarFormat, type RarEntry, type RarInfo } from './parse.js'
+
+/** Reads the volumes of a RAR set, a segment at a time. */
+export interface VolumeReader {
+  readonly volumes: number
+  /**
+   * Bytes of volume `v` from `offset` to the end of the segment that holds
+   * it; with `more`, the following segment too (for a header cut off by the
+   * segment end). Null when that can't be read (missing article, budget).
+   */
+  read(v: number, offset: number, more: boolean): Promise<Buffer | null>
+  /** Size of volume `v`, known once something of it was read. */
+  size(v: number): number | null
+}
+
+export interface RarWalk {
+  format: 'rar4' | 'rar5' | null
+  /** One entry per name, from its first header. */
+  entries: RarEntry[]
+  /** Every volume was walked to its end. */
+  complete: boolean
+  headersEncrypted: boolean
+  /** Why the walk ended early. */
+  stopped?: 'unreadable' | 'invalid' | 'encrypted'
+}
+
+/**
+ * Lists a RAR set completely by hopping from header to header (CONCEPT.md
+ * D40). Each header says how much data follows it, so the next header's
+ * offset is known and only the segment holding it is read: one segment per
+ * entry and one per volume, instead of the whole set. Split files repeat
+ * their header at the start of every volume they continue in.
+ */
+export async function walkRar(reader: VolumeReader): Promise<RarWalk> {
+  const seen = new Map<string, RarEntry>()
+  const result = (stopped?: RarWalk['stopped']): RarWalk => ({
+    format,
+    entries: [...seen.values()],
+    complete: !stopped,
+    headersEncrypted: stopped === 'encrypted',
+    stopped,
+  })
+  let format: 'rar4' | 'rar5' | null = null
+  let v = 0
+  let offset = 0
+
+  while (v < reader.volumes) {
+    let info: RarInfo | null = null
+    let reached = -1
+    for (const more of [false, true]) {
+      const data = await reader.read(v, offset, more)
+      if (!data) return result('unreadable')
+      if (offset === 0) format = rarFormat(data) ?? format
+      info = offset === 0 ? parseRarHeaders(data) : format && parseRarHeaders(data, format)
+      if (!info) return result('invalid')
+      // Stopped inside the data: a header the segment end cut off (read on), or a bad one.
+      if (!info.truncated || info.next >= data.length || info.headersEncrypted) break
+      // Read on and got further: the next header is cut off at the end of the following segment.
+      if (more && info.next > reached) break
+      if (more) return result('invalid')
+      reached = info.next
+    }
+    if (info!.headersEncrypted) return result('encrypted')
+    for (const entry of info!.entries) if (!seen.has(entry.name)) seen.set(entry.name, entry)
+
+    const next = offset + info!.next
+    const size = reader.size(v)
+    // The end-of-archive header closes every volume; data running to the end does too.
+    if (!info!.truncated || size === null || next >= size) {
+      v++
+      offset = 0
+    } else {
+      offset = next
+    }
+  }
+  return result()
+}

lib/seed/plan.ts

@@ -0,0 +1,27 @@
+/**
+ * Seed planning arithmetic (CONCEPT.md D38), independent of the database.
+ */
+
+/** The parts of `from`..`to` (inclusive) that no range in `covered` (inclusive pairs) overlaps, oldest first. */
+export function uncovered(from: number, to: number, covered: [number, number][]): [number, number][] {
+  const gaps: [number, number][] = []
+  let next = from
+  for (const [start, end] of [...covered].sort((a, b) => a[0] - b[0])) {
+    if (end < next) continue
+    if (start > to) break
+    if (start > next) gaps.push([next, Math.min(start - 1, to)])
+    next = Math.max(next, end + 1)
+    if (next > to) return gaps
+  }
+  if (next <= to) gaps.push([next, to])
+  return gaps
+}
+
+/** `gaps` cut into ranges of at most `size` articles, newest first (from the top of the newest gap down). */
+export function cut(gaps: [number, number][], size: number): [number, number][] {
+  const ranges: [number, number][] = []
+  for (const [from, to] of [...gaps].sort((a, b) => b[1] - a[1])) {
+    for (let end = to; end >= from; end -= size) ranges.push([Math.max(from, end - size + 1), end])
+  }
+  return ranges
+}

lib/seed/runner.ts

@@ -39,6 +39,11 @@ export async function runRange(options: RunRangeOptions): Promise<void> {
   const idle = options.idle ?? 5
   const submitEvery = options.submitEvery ?? 5
   const aggregator = new FileAggregator<IngestSegment>()
+  // A range fully reported before but not finished (its node went away right at the end) resumes past its end.
+  if (options.start > options.end) {
+    await options.sink(emptyBatch(options.end + 1))
+    return
+  }
   await options.client.group(options.group)
 
   const fetch = (from: number) => {

lib/sevenzip/index.ts

@@ -12,8 +12,13 @@ import { crc32 } from 'node:zlib'
 const SIGNATURE = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])
 const SIGNATURE_HEADER_SIZE = 32
 /** Header property ids (7zFormat.txt). */
-const K_ENCODED_HEADER = 0x17
+const K_END = 0x00
 const K_PACK_INFO = 0x06
+const K_UNPACK_INFO = 0x07
+const K_SIZE = 0x09
+const K_CRC = 0x0a
+const K_FOLDER = 0x0b
+const K_ENCODED_HEADER = 0x17
 
 export interface SevenZipStart {
   /** Where the archive header starts, from the start of the archive. */
@@ -49,6 +54,68 @@ export function listingStart(start: SevenZipStart, header: Buffer): number {
   return SIGNATURE_HEADER_SIZE + packPos
 }
 
+/** 7zAES, the coder of password-protected streams. */
+const AES_CODER = Buffer.from([0x06, 0xf1, 0x07, 0x01])
+
+/**
+ * Whether an encoded header (`header`, the bytes at headerStart) is
+ * encrypted (7z -mhe): its coders include 7zAES. Then nothing can be listed
+ * without the password, and neither the packed header nor bsdtar is needed.
+ * Null when the header isn't encoded or doesn't read as expected.
+ */
+export function encodedHeaderEncrypted(header: Buffer): boolean | null {
+  try {
+    if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return null
+    let pos = 2
+    let numPackStreams: number
+    ;[, pos] = readNumber(header, pos) // packPos
+    ;[numPackStreams, pos] = readNumber(header, pos)
+    for (let id = header[pos++]; id !== K_END; id = header[pos++]) {
+      if (id === K_SIZE) {
+        for (let i = 0; i < numPackStreams; i++) [, pos] = readNumber(header, pos)
+      } else if (id === K_CRC) {
+        const allDefined = header[pos++]
+        let defined = numPackStreams
+        if (!allDefined) {
+          const bits = header.subarray(pos, pos + Math.ceil(numPackStreams / 8))
+          pos += bits.length
+          defined = [...bits].reduce((sum, byte) => sum + byte.toString(2).replaceAll('0', '').length, 0)
+        }
+        pos += 4 * defined
+      } else {
+        return null
+      }
+      if (pos >= header.length) return null
+    }
+    if (header[pos++] !== K_UNPACK_INFO || header[pos++] !== K_FOLDER) return null
+    let numFolders: number
+    ;[numFolders, pos] = readNumber(header, pos)
+    if (numFolders < 1 || header[pos++] !== 0) return null // folders stored elsewhere
+    let numCoders: number
+    ;[numCoders, pos] = readNumber(header, pos)
+    if (!(numCoders >= 1) || pos >= header.length) return null
+    for (let c = 0; c < numCoders; c++) {
+      const flags = header[pos++]
+      const idSize = flags & 0x0f
+      if (header.subarray(pos, pos + idSize).equals(AES_CODER)) return true
+      pos += idSize
+      if (flags & 0x10) {
+        ;[, pos] = readNumber(header, pos)
+        ;[, pos] = readNumber(header, pos)
+      }
+      if (flags & 0x20) {
+        let size: number
+        ;[size, pos] = readNumber(header, pos)
+        pos += size
+      }
+      if (pos > header.length) return null
+    }
+    return false
+  } catch {
+    return null
+  }
+}
+
 /**
  * 7z's variable-length number: the leading 1-bits of the first byte count
  * the little-endian bytes that follow; its remaining bits are the high part.

lib/sevenzip/list.ts

@@ -0,0 +1,171 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, open, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { decodeSegment, type DecodedSegment } from '../yenc/index.js'
+import type { RarInfo } from '../rar/parse.js'
+import type { BodyFetcher } from '../rar/segment_reader.js'
+import {
+  checkHeader,
+  encodedHeaderEncrypted,
+  listingStart,
+  parseBsdtarListing,
+  parseSevenZipStart,
+  type SevenZipStart,
+} from './index.js'
+
+const run = promisify(execFile)
+
+/** The tail read for a listing: header plus packed header; more is not a header. */
+const MAX_TAIL_BYTES = 16 * 1024 * 1024
+
+/**
+ * Lists a 7z archive, also a split one (.7z.001, .002, …), from the first
+ * segment of the first volume (`firstData`, decoded) and the last segment(s)
+ * of the last one (CONCEPT.md D19). Those bytes are put at their place in a
+ * sparse file the size of the whole archive, which bsdtar (libarchive)
+ * lists: it only reads the start and the header at the end. Null when it
+ * isn't a 7z start, or the tail doesn't hold the header its start points to.
+ */
+export async function listSevenZip(
+  firstData: Buffer,
+  volumeCount: number,
+  lastVolume: { number: number; id: string }[],
+  fetch: BodyFetcher,
+  warn: (message: string) => void = () => {}
+): Promise<RarInfo | null> {
+  const start = parseSevenZipStart(firstData)
+  if (!start) return null
+  const split = volumeCount > 1
+  if (start.headerSize === 0) return sevenZipInfo([], false, false, split)
+
+  const tail = await readTail(lastVolume, start, fetch)
+  if (!tail) return null
+  // Encrypted headers (-mhe): known from the header alone, no listing possible (most 7z uploads).
+  if ('headersEncrypted' in tail) return sevenZipInfo([], true, true, split)
+
+  const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
+  try {
+    const archive = path.join(dir, 'archive.7z')
+    const handle = await open(archive, 'w')
+    try {
+      await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
+      await handle.write(tail.data, 0, tail.data.length, tail.from)
+      await handle.truncate(start.archiveSize)
+    } finally {
+      await handle.close()
+    }
+    return await bsdtar(archive, split, warn)
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}
+
+/** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
+async function bsdtar(archive: string, split: boolean, warn: (message: string) => void): Promise<RarInfo | null> {
+  let stdout = ''
+  try {
+    stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
+  } catch (error: any) {
+    if (/header is encrypted/i.test(String(error.stderr))) return sevenZipInfo([], true, true, split)
+    warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
+    return null
+  }
+  const entries = parseBsdtarListing(stdout)
+  const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
+  let encrypted = false
+  if (sample) {
+    try {
+      await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
+    } catch (error: any) {
+      // Without the data (it's all zeros here), anything else fails too: only this message counts.
+      encrypted = /content is encrypted/i.test(String(error.stderr))
+    }
+  }
+  return sevenZipInfo(entries, false, encrypted, split)
+}
+
+export function sevenZipInfo(
+  entries: { name: string; size: number; directory: boolean }[],
+  headersEncrypted: boolean,
+  encrypted: boolean,
+  split: boolean
+): RarInfo {
+  return {
+    format: '7z',
+    volume: split,
+    firstVolume: true,
+    solid: false,
+    headersEncrypted,
+    truncated: false,
+    next: 0,
+    entries: entries.map((e) => ({
+      name: e.name,
+      size: e.size,
+      packedSize: 0,
+      directory: e.directory,
+      encrypted: encrypted && !e.directory,
+      splitBefore: false,
+      splitAfter: false,
+      stored: false,
+      dataOffset: 0,
+    })),
+  }
+}
+
+/**
+ * The archive's tail from the last volume: the header (checked against its
+ * CRC) and, for an encoded header, the packed stream before it. `from` is
+ * its position in the whole archive.
+ */
+async function readTail(
+  last: { number: number; id: string }[],
+  start: SevenZipStart,
+  fetch: BodyFetcher
+): Promise<{ from: number; data: Buffer } | { headersEncrypted: true } | null> {
+  const ids = new Map(last.map((s) => [s.number, s]))
+  const numbers = [...ids.keys()]
+  if (numbers.length === 0) return null
+  const segments = new Map<number, DecodedSegment>()
+  const segment = async (n: number) => {
+    if (!segments.has(n)) {
+      const id = ids.get(n)?.id
+      if (!id) throw new Error(`part ${n} of the last 7z volume is missing`)
+      segments.set(n, decodeSegment(await fetch(id), { lenientSize: true }))
+    }
+    return segments.get(n)!
+  }
+
+  const lastNumber = numbers[numbers.length - 1]
+  const lastSegment = await segment(lastNumber)
+  const volumeSize = lastSegment.meta.begin.size
+  const volumeStart = start.archiveSize - volumeSize
+  const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
+  const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
+  if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
+
+  // Bytes [from, to) of the whole archive, from the last volume's parts.
+  const read = async (from: number, to: number): Promise<Buffer | null> => {
+    if (from < volumeStart) return null
+    const a = from - volumeStart
+    const b = to - volumeStart
+    const pieces: Buffer[] = []
+    for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
+      const s = await segment(n)
+      const offset = (s.meta.part?.begin ?? 1) - 1
+      if (offset !== (n - 1) * partSize) return null
+      pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
+    }
+    const data = Buffer.concat(pieces)
+    return data.length === b - a ? data : null
+  }
+
+  const header = await read(start.headerStart, start.archiveSize)
+  if (!header || !checkHeader(start, header)) return null
+  if (encodedHeaderEncrypted(header)) return { headersEncrypted: true }
+  const from = Math.min(listingStart(start, header), start.headerStart)
+  if (start.archiveSize - from > MAX_TAIL_BYTES) return null
+  const data = await read(from, start.archiveSize)
+  return data ? { from, data } : null
+}

lib/spotnet/password.ts

@@ -3,14 +3,25 @@
  * the NZB's `<meta type="password">`:
  *   `Title{{secret}}` (the newznab/SABnzbd convention),
  *   `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`,
- *   `WW FC0401` (abbreviation without separator: only with a digit in it).
+ *   a few words between keyword and separator (`Het wachtwoord voor de nzb is: dw0067`,
+ *   `ww in nzb: X`, `Wachtwoord inbegrepen: wysiwyg`),
+ *   the value on the next line (`WW:` / `tijdloos`),
+ *   `wachtwoord is arn2`, `… voorzien van een wachtwoord. Deze is: X`,
+ *   `WW FC0401` (abbreviation without separator: with a digit in it, or as the whole line).
  * The value must be the rest of its line and a single token, so sentences
  * such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or
- * "met een password beschermd" don't count.
+ * "met een password beschermd" don't count (LOG.md 2026-10-08 (10)).
  */
-const KEYWORD = String.raw`wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw`
-const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])(?:${KEYWORD})\s*(?:is\s*)?[:=]\s*(.*)$`, 'i')
+const KEYWORD = String.raw`(?:wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw)(?![a-z])`
+const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}(?:\s+[a-zà-ÿ]+){0,3}?\s*(?:is\s*)?[:=]\s*(.*)$`, 'i')
+const IS_VALUE = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}\s+is\s+(\S+)\s*$`, 'i')
+/** "Deze is voorzien van een wachtwoord. Deze is: X": the value in the next sentence. */
+const NEXT_SENTENCE = /(?:^|[.!?]\s*)(?:deze|dit|het|die|this|it)\s+is\s*[:=]\s*(\S+)\s*$/i
+const MENTIONS = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}`, 'i')
 const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i
+/** "WW carrie" as a line of its own; "WW II" is the war. */
+const WHOLE_LINE_ABBREVIATION = /^(?:ww|pw)\s+(\S+)$/i
+const NOT_PASSWORD_TOKEN = /^(?:i{1,3}|iv|v|[12])$/i
 const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i
 const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i
 
@@ -24,7 +35,11 @@ function lines(description: string): string[] {
 }
 
 function clean(value: string): string | null {
-  const token = value.trim().replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '')
+  const token = value
+    .trim()
+    // "Wachtwoord : yakowyke !!"
+    .replace(/^[!*~#]+\s*|\s*[!*~#]+$/g, '')
+    .replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '')
   if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null
   return token
 }
@@ -32,19 +47,37 @@ function clean(value: string): string | null {
 export function passwordFromSpot(title: string, description?: string | null): string | null {
   const braces = /\{\{([^{}]{1,64})\}\}/.exec(title)
   if (braces && clean(braces[1])) return clean(braces[1])
-  for (const line of lines(description ?? '')) {
+  const all = lines(description ?? '')
+  for (const [i, line] of all.entries()) {
     if (!line || NO_PASSWORD.test(line)) continue
     const separated = WITH_SEPARATOR.exec(line)
     if (separated) {
-      const value = clean(separated[1])
+      const value = separated[1].trim() ? clean(separated[1]) : nextLineValue(all, i)
       if (value) return value
       continue
     }
-    const abbreviated = ABBREVIATION.exec(line)
-    if (abbreviated && /\d/.test(abbreviated[1])) {
+    const is = IS_VALUE.exec(line)
+    if (is) {
+      const value = clean(is[1])
+      if (value) return value
+    }
+    const sentence = MENTIONS.test(line) ? NEXT_SENTENCE.exec(line) : null
+    if (sentence) {
+      const value = clean(sentence[1])
+      if (value) return value
+    }
+    const whole = WHOLE_LINE_ABBREVIATION.exec(line)
+    const abbreviated = whole ?? ABBREVIATION.exec(line)
+    if (abbreviated && (whole ? !NOT_PASSWORD_TOKEN.test(abbreviated[1]) : /\d/.test(abbreviated[1]))) {
       const value = clean(abbreviated[1])
       if (value) return value
     }
   }
   return null
 }
+
+/** `WW:` with the value alone on the next non-empty line. */
+function nextLineValue(all: string[], i: number): string | null {
+  const next = all.slice(i + 1).find((line) => line !== '')
+  return next ? clean(next) : null
+}

seed-node/README.md

@@ -5,8 +5,14 @@ D38). It asks the seed API for a range over HTTPS, gets that range's headers
 (by default through the seed API, which fetches them from the news server and
 passes them on gzip-compressed), groups them into files, and posts the
 results back. It only makes outgoing connections: no port forwarding,
-database, Redis or news server account. Only a token and
-`SEED_CONNECTIONS` need setting.
+database, Redis or news server account. Only the token needs setting, plus
+how much work to take on.
+
+A node can also do content inspection and archive walks (CONCEPT.md D40,
+D41) with `SEED_INSPECTORS`: it takes files from the indexer's inspection
+queue, downloads their first segment(s) through the seed API, works out
+what they are (par2 sets, RAR and 7z listings, NZBs, passwords) and posts
+the findings back; the indexer only stores them.
 
 Headers through the seed API (`SEED_FETCH=relay`, the default) use the
 indexer's news server account: `SEED_NNTP_CONNECTIONS` in the indexer's
@@ -20,10 +26,11 @@ the NNTP lines in `node.env`).
 - Outgoing HTTPS to `seed.exp.sforum.nl` (the seed API) and
   `gitea.exp.sforum.nl` (to build the image), and outgoing NNTP to the news
   server (port 563 for TLS).
-- Per connection: roughly 0.4 CPU core and 1–3 MB/s download (headers come
+- Per range seeded at once: roughly 0.4 CPU core and 1–3 MB/s download (headers come
   gzip-compressed); a few hundred MB of RAM per node. Disk: about 1.5 GB for
-  the image.
-- An NNTP account. Every connection a node opens counts against that
+  the image. Per inspector: up to a core while decoding, and a few MB/s
+  download (whole articles, ~700 KB each).
+- Only with `SEED_FETCH=direct`: an NNTP account. Every connection a node opens counts against that
   account's connection limit, next to the ~20 the indexer itself uses if it's
   the same account. Too many gives `481`/`502` errors at login. Some
   providers also limit the number of IP addresses at once (Eweka: 2, as
@@ -55,13 +62,13 @@ for the same name replaces the token; `--disable` revokes it.
 |---|---|
 | `SEED_URL` | `https://seed.exp.sforum.nl` (leave as is) |
 | `SEED_TOKEN` | the token from step 1 |
-| `SEED_CONNECTIONS` | ranges worked on at once, one NNTP connection each. Start with 2–4. |
-| `NNTP_HOST`, `NNTP_PORT`, `NNTP_SECURE` | the news server; over the internet use TLS: port `563`, `NNTP_SECURE=true` (for Eweka: `news.eweka.nl`) |
-| `NNTP_USER`, `NNTP_PASSWORD` | the NNTP account |
-| `NNTP_TLS_SERVERNAME` | only through a tunnel: the news server's real hostname, for the TLS certificate check |
+| `SEED_RANGES` | ranges seeded at once. Start with 2–4; `0` for no seeding. |
+| `SEED_INSPECTORS` | inspections and archive walks at once (default `0`). Try 2–4 on a machine with spare cores. |
+| `SEED_PROCESSES` | optional: how many processes the ranges and inspections are spread over, one CPU core each. Default: the number of cores (never more than ranges + inspections). Lower it to leave cores free. |
+| `SEED_FETCH=direct` and `NNTP_*` | optional: seed with this node's own NNTP account (see the commented lines). `NNTP_TLS_SERVERNAME` only through a tunnel: the news server's real hostname, for the TLS certificate check. |
 
-Leave the block below "Required by the app's configuration, but unused" as
-it is: the app checks these settings at startup, but a node never uses them.
+Nothing else: the database, Redis and web server settings the indexer
+needs are filled in for a node automatically.
 
 ## 4. Start it
 
@@ -70,7 +77,7 @@ it is: the app checks these settings at startup, but a node never uses them.
 
 The first build takes a few minutes. A working node logs:
 
-    [ info ] Seeding node for https://seed.exp.sforum.nl with 2 connection(s).
+    [ info ] Seeding node for https://seed.exp.sforum.nl: 2 range(s) at once, headers through the seed API; 2 inspection(s) at once, articles through the seed API.
     [ info ] [1] alt.binaries.ath 49834000000–49835999999
     [ info ] [1] done: 2,000,000 headers in 237 s (8,450/s)
 
@@ -105,7 +112,8 @@ articles reached, and per node its leases and headers.
 |---|---|
 | `The seed API rejected this node token` | wrong or revoked `SEED_TOKEN`; the node stops |
 | `Seed API …: 404 …; retrying` | the seed API is restarting (a deploy on homelab); the node retries |
-| `NNTP login failed: …` | the node didn't take a range and tries again in a minute (5 minutes for the IP-address limit); `481`/`502`: wrong account, too many connections (lower `SEED_CONNECTIONS`) or too many IP addresses |
+| `NNTP login failed: …` | the node didn't take a range and tries again in a minute (5 minutes for the IP-address limit); `481`/`502`: wrong account, too many connections (lower `SEED_RANGES`) or too many IP addresses |
 | `Hostname/IP does not match certificate's altnames: Host: localhost` | TLS through a tunnel: set `NNTP_TLS_SERVERNAME` to the news server's hostname |
 | `Range … failed: …` | the range is given back and retried later, possibly by another node |
 | `Range … passed to another node; dropping it.` | this node was too slow to report (15 minutes); harmless |
+| `[inspect n] "…" failed: …` | an inspection failed (often a missing article); the indexer retries it later |

seed-node/node.env.example

@@ -1,34 +1,20 @@
-# --- Seed API: the orchestrator, and this node's token (`node ace seed:token <name>` on the indexer).
+# The seed API (the orchestrator) and this node's token (`node ace seed:token <name>` on the indexer).
 SEED_URL=https://seed.exp.sforum.nl
 SEED_TOKEN=sn_replace_me
-# Ranges worked on at once (about 0.4 CPU core each).
-SEED_CONNECTIONS=4
-# relay: headers come through the seed API, no news server account needed (the default).
-# direct: the node fetches with its own NNTP account; then fill in the NNTP_* lines below.
-SEED_FETCH=relay
 
-# --- Only for SEED_FETCH=direct: the NNTP account (counts against its connection and IP limits).
-NNTP_HOST=localhost
-NNTP_PORT=563
-NNTP_SECURE=true
-NNTP_USER=unused
-NNTP_PASSWORD=unused
+# Ranges seeded at once (about 0.4 CPU core each). 0: no seeding.
+SEED_RANGES=4
+# Content inspections and archive walks at once (CPU for decoding and parsing; articles come through the seed API).
+SEED_INSPECTORS=0
+# Ranges and inspections are spread over this many processes, one core each (default: the number of cores).
+# SEED_PROCESSES=4
+
+# Optional: SEED_FETCH=direct seeds with this node's own NNTP account instead of through the seed API.
+# SEED_FETCH=direct
+# NNTP_HOST=news.eweka.nl
+# NNTP_PORT=563
+# NNTP_SECURE=true
+# NNTP_USER=
+# NNTP_PASSWORD=
 # Through a tunnel or proxy: the hostname the server's TLS certificate is for.
 # NNTP_TLS_SERVERNAME=news.eweka.nl
-
-# --- Required by the app's configuration, but unused on a seeding node (no database, Redis or web server).
-NODE_ENV=production
-TZ=UTC
-PORT=3333
-HOST=0.0.0.0
-LOG_LEVEL=info
-APP_KEY=unused-on-seed-nodes-unused-on-seed-nodes
-APP_URL=http://localhost:3333
-SESSION_DRIVER=memory
-DB_CONNECTION=pg
-DB_HOST=localhost
-DB_PORT=5432
-DB_USER=unused
-DB_DATABASE=unused
-REDIS_HOST=localhost
-REDIS_PORT=6379

start/env.ts

@@ -11,6 +11,38 @@
 
 import { Env } from '@adonisjs/core/env'
 
+/**
+ * A seeding node (`node ace seed:node`, CONCEPT.md D38) has no database,
+ * Redis or web server, and in relay mode no NNTP account, but the app's
+ * configuration still reads those settings: they get placeholders here, so
+ * a node needs only SEED_URL and SEED_TOKEN. Values that are set win (e.g.
+ * the NNTP account of a node with SEED_FETCH=direct).
+ */
+if (process.env.ACE_COMMAND === 'seed:node') {
+  const placeholders: Record<string, string> = {
+    NODE_ENV: 'production',
+    PORT: '3333',
+    HOST: '0.0.0.0',
+    LOG_LEVEL: 'info',
+    APP_KEY: 'unused-on-seed-nodes-unused-on-seed-nodes',
+    APP_URL: 'http://localhost:3333',
+    SESSION_DRIVER: 'memory',
+    DB_CONNECTION: 'pg',
+    DB_HOST: 'localhost',
+    DB_PORT: '5432',
+    DB_USER: 'unused',
+    DB_DATABASE: 'unused',
+    REDIS_HOST: 'localhost',
+    REDIS_PORT: '6379',
+    NNTP_HOST: 'localhost',
+    NNTP_PORT: '563',
+    NNTP_USER: 'unused',
+    NNTP_PASSWORD: 'unused',
+    NNTP_SECURE: 'true',
+  }
+  for (const [name, value] of Object.entries(placeholders)) process.env[name] ??= value
+}
+
 export default await Env.create(new URL('../', import.meta.url), {
   // Node
   NODE_ENV: Env.schema.enum(['development', 'production', 'test'] as const),
@@ -50,6 +82,8 @@ export default await Env.create(new URL('../', import.meta.url), {
   NNTP_COMMAND_TIMEOUT_MS: Env.schema.number.optional(),
   NNTP_TLS_SERVERNAME: Env.schema.string.optional(),
   SEED_NNTP_CONNECTIONS: Env.schema.number.optional(),
+  SEED_NNTP_TIMEOUT_MS: Env.schema.number.optional(),
+  SEED_NNTP_COMPRESS: Env.schema.boolean.optional(),
 
   // Indexer (config/indexer.ts)
   BACKFILL_MAX_QUEUED: Env.schema.number.optional(),
@@ -68,6 +102,16 @@ export default await Env.create(new URL('../', import.meta.url), {
   FETCH_CONCURRENCY: Env.schema.number.optional(),
   BODY_CONCURRENCY: Env.schema.number.optional(),
   COLLECTION_CONCURRENCY: Env.schema.number.optional(),
+  ARCHIVE_WALK_CONCURRENCY: Env.schema.number.optional(),
+  RELEASE_COMPLETE_INTERVAL_MS: Env.schema.number.optional(),
+  FILE_KEYS_FILL_SECONDS: Env.schema.number.optional(),
+  GC_AWAIT_INSPECTION_SECONDS: Env.schema.number.optional(),
+  GC_MERGE_INTERVAL_MS: Env.schema.number.optional(),
+  INSPECTION_FEED_MAX: Env.schema.number.optional(),
+  GARBAGE_FILTER: Env.schema.boolean.optional(),
+  GARBAGE_EXPLORE_PERCENT: Env.schema.number.optional(),
+  GARBAGE_LEARN_INTERVAL_MS: Env.schema.number.optional(),
+  ARCHIVE_WALK_MAX_SEGMENTS: Env.schema.number.optional(),
   DEFERRED_SETTLE_MS: Env.schema.number.optional(),
   DEFERRED_EXPAND_MAX_PARTS: Env.schema.number.optional(),
   DEFERRED_CONCURRENCY: Env.schema.number.optional(),

start/routes.ts

@@ -9,12 +9,20 @@
 
 import { middleware } from '#start/kernel'
 import router from '@adonisjs/core/services/router'
+import db from '@adonisjs/lucid/services/db'
 import { controllers } from '#generated/controllers'
 
 const ReleasesController = () => import('#controllers/releases_controller')
 const NewznabController = () => import('#controllers/newznab_controller')
 const SeedController = () => import('#controllers/seed_controller')
 
+// Docker health checks (web, seed-api, seed-relay): the database answers, nothing more. They called
+// /api/v1/stats every 30 s from five containers, and its query kept a core busy (LOG.md 2026-10-07 (5)).
+router.get('/health', async () => {
+  await db.rawQuery('SELECT 1')
+  return { ok: true }
+})
+
 // Newznab API for Prowlarr / Sonarr / Radarr ("Generic Newznab", API path /api).
 router.get('/api', [NewznabController, 'handle'])
 
@@ -59,6 +67,12 @@ router
     router.post('ranges/:id/done', [SeedController, 'done']).where('id', router.matchers.number())
     router.post('ranges/:id/release', [SeedController, 'release']).where('id', router.matchers.number())
     router.get('status', [SeedController, 'status'])
+    // Content inspection and archive walks on nodes (CONCEPT.md D41).
+    router.post('inspect/lease', [SeedController, 'inspectLease'])
+    router.post('inspect/done', [SeedController, 'inspectDone'])
+    router.post('inspect/failed', [SeedController, 'inspectFailed'])
+    router.post('inspect/release', [SeedController, 'inspectRelease'])
+    router.get('articles', [SeedController, 'article'])
   })
   .prefix('/api/v1/seed')
   .use(middleware.seedNode())

tests/functional/binary_files.spec.ts

@@ -0,0 +1,86 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import IngestService, { type IngestSegment } from '#services/IngestService'
+import BinaryFileService from '#services/BinaryFileService'
+import QueueService from '#services/QueueService'
+import FileStore, { segmentsOf } from '#services/FileStore'
+import { fileKey } from '#lib/subject/index'
+import ContentService from '#services/ContentService'
+
+function segment(name: string, number: number, total = 3): IngestSegment {
+  const poster = 'batch-test@example.invalid'
+  const subject = `"${name}" yEnc`
+  return { fileKey: fileKey(poster, subject, total), poster, subject, name, totalParts: total, number, messageId: `${name}-${number}@test`, bytes: 100, groups: ['alt.binaries.test'] }
+}
+
+const binary = (name: string, total = 3) => db.from('binaries').where('file_key', segment(name, 1, total).fileKey).first()
+
+test.group('Binaries to files', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.teardown(() => QueueService.closeAll())
+
+  test('ingest counts stored parts, once each, and claims a binary by that count', async ({ assert }) => {
+    await IngestService.ingest([segment('a.bin', 1), segment('a.bin', 2)])
+    // The same article again (a crosspost): not counted twice.
+    await IngestService.ingest([segment('a.bin', 2)])
+    assert.equal((await binary('a.bin')).part_count, 2)
+    assert.isNull((await binary('a.bin')).completed_at)
+    const completed = await IngestService.ingest([segment('a.bin', 3)])
+    assert.equal((await binary('a.bin')).part_count, 3)
+    assert.deepEqual(completed.map((c) => c.fileKey), [segment('a.bin', 1).fileKey])
+  })
+
+  test('a binary from before the count is counted exactly on its next part', async ({ assert }) => {
+    await IngestService.ingest([segment('b.bin', 1), segment('b.bin', 2)])
+    await db.from('binaries').where('id', (await binary('b.bin')).id).update({ part_count: null })
+    const completed = await IngestService.ingest([segment('b.bin', 3)])
+    assert.equal((await binary('b.bin')).part_count, 3)
+    assert.lengthOf(completed, 1)
+  })
+
+  test('stores a batch: new files, existing ones linked, incomplete ones left for the retry', async ({ assert }) => {
+    await IngestService.ingest([1, 2, 3].flatMap((n) => [segment('new.bin', n), segment('known.bin', n)]))
+    await IngestService.ingest([segment('half.bin', 1)])
+    // known.bin is a file already (e.g. from an NZB), found by its first segment.
+    const now = new Date()
+    const [known] = await FileStore.insert([
+      { filename: 'known.bin', poster: 'x', date: now.getTime(), parts: 3, groups: '[]', segments: [1, 2, 3].map((n) => ({ number: n, id: `known.bin-${n}@test`, size: 100 })), created_at: now, updated_at: now },
+    ])
+    const ids = [await binary('new.bin'), await binary('known.bin'), await binary('half.bin')].map((b) => Number(b.id))
+
+    const result = await BinaryFileService.store(ids)
+    assert.include(result, { saved: 1, existing: 1 })
+    assert.deepEqual(result.incomplete, [ids[2]])
+
+    const stored = await db.from('files').where('file_key', segment('new.bin', 1).fileKey).first()
+    assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
+    assert.equal(Number((await binary('new.bin')).file_id), Number(stored.id))
+    assert.equal(Number((await binary('known.bin')).file_id), known.id)
+    // Header parts are in the file's segment list now.
+    const [{ n }] = await db.from('parts').whereIn('binary_id', [ids[0], ids[1]]).count('* as n')
+    assert.equal(Number(n), 0)
+
+    // The retry: done ones are skipped, the incomplete one again incomplete.
+    const again = await BinaryFileService.store(ids)
+    assert.include(again, { saved: 0, skipped: 2 })
+    assert.deepEqual(again.incomplete, [ids[2]])
+  })
+
+  test('a queued obfuscated file keeps its binary until inspection fingerprints it', async ({ assert }) => {
+    const name = 'Zk3m9QpL2xW8rT5vN1bYq.bin'
+    await IngestService.ingest([1, 2, 3].map((n) => segment(name, n)))
+    const b = await binary(name)
+    const result = await BinaryFileService.store([Number(b.id)])
+    assert.include(result, { saved: 1, inspections: 1 })
+    const waiting = await binary(name)
+    assert.isTrue(waiting.awaiting_inspection)
+
+    // Inspection stores the fingerprint and clears the flag.
+    const task = { kind: 'inspect' as const, fileId: Number(waiting.file_id), name, segments: [], sevenZip: null }
+    await ContentService.apply(task, { size: 300, contentType: null, md5_16k: 'a'.repeat(32), yencName: name })
+    const inspected = await binary(name)
+    assert.isNull(inspected.awaiting_inspection)
+    assert.equal(inspected.md5_16k, 'a'.repeat(32))
+  })
+})

tests/functional/file_keys.spec.ts

@@ -0,0 +1,41 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileKeys from '#services/FileKeys'
+import FileStore from '#services/FileStore'
+
+const file = (key: string, name: string) => {
+  const now = new Date()
+  return { file_key: key, filename: name, poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [], created_at: now, updated_at: now }
+}
+
+test.group('file_keys', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.each.teardown(() => FileKeys.forget())
+
+  test('the fill copies keys from before, then file inserts use the filter', async ({ assert }) => {
+    // A file from before file_keys: in files, not in file_keys.
+    const [old] = await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'old.bin')])
+    await db.rawQuery(`DELETE FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-old'.padEnd(40, '0')])
+    await db.from('file_keys_fill').where('id', 1).update({ next_id: 0, until_id: null, complete: false })
+    FileKeys.forget()
+
+    // Not complete: every partition is checked, so the old key is found anyway.
+    assert.isFalse(await FileKeys.complete())
+    assert.lengthOf(await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'again.bin')]), 0)
+
+    const filled = await FileKeys.fill(60_000)
+    assert.isTrue(filled.complete)
+    assert.isAtLeast(filled.untilId, old.id)
+    const [{ n }] = (await db.rawQuery(`SELECT count(*) AS n FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-old'.padEnd(40, '0')])).rows
+    assert.equal(Number(n), 1)
+
+    // Complete: the filter says the old key may exist, the partitions confirm it; a new key goes in.
+    assert.isTrue(await FileKeys.complete())
+    assert.lengthOf(await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'again.bin')]), 0)
+    assert.lengthOf(await FileStore.insert([file('fk-new'.padEnd(40, '0'), 'new.bin')]), 1)
+    // And the trigger put the new one in file_keys.
+    const [{ m }] = (await db.rawQuery(`SELECT count(*) AS m FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-new'.padEnd(40, '0')])).rows
+    assert.equal(Number(m), 1)
+  })
+})

tests/functional/file_partitions.spec.ts

@@ -39,10 +39,11 @@ test.group('files partitions', (group) => {
       )
       return Number(rows[0].upper)
     }
+    await FilePartitions.ensure()
     const before = await upper()
     assert.deepEqual(await FilePartitions.ensure(), [])
-    // A file near the end of the covered ids: one more partition is needed.
-    await db.rawQuery(`SELECT setval('files_id_seq', ?, false)`, [before - FILE_PARTITION_SIZE - 10])
+    // Ids handed out up to just inside the four partitions kept ahead: one more is needed.
+    await db.rawQuery(`SELECT setval('files_id_seq', ?, false)`, [before - 4 * FILE_PARTITION_SIZE + 10])
     await FileStore.insert([file('e'.repeat(40), 'late.bin')])
     assert.deepEqual(await FilePartitions.ensure(), [`files_p${before}`])
     assert.equal(await upper(), before + FILE_PARTITION_SIZE)

tests/functional/garbage.spec.ts

@@ -0,0 +1,190 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import indexerConfig from '#config/indexer'
+import FileStore from '#services/FileStore'
+import SeedService from '#services/SeedService'
+import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
+import ContentService from '#services/ContentService'
+import GarbageFilter from '#services/GarbageFilter'
+import QueueService from '#services/QueueService'
+import { FileAggregator } from '#lib/aggregate/index'
+import { toSeedFile } from '#lib/seed/wire'
+import { explored, uploadTool } from '#lib/garbage/index'
+
+const GROUP = 'alt.binaries.test'
+let n = 0
+
+/** Headers of a file posted by ngPost (or nyuu), all its parts. */
+function upload(name: string, parts: number, tool: 'ngPost' | 'nyuu' = 'ngPost') {
+  const id = () => {
+    n++
+    return tool === 'ngPost' ? `${n.toString(16).padStart(32, '0')}@ngPost` : `${'FmFwIaTpGgUiSfYcMwFl'}${String.fromCharCode(65 + (n % 26))}e${String.fromCharCode(65 + ((n >> 5) % 26))}o-${String(1790637260000 + n)}@nyuu`
+  }
+  return Array.from({ length: parts }, (_, i) =>
+    segmentFromHeader(
+      {
+        'subject': `[1/1] - "${name}" yEnc (${i + 1}/${parts})`,
+        'from': 'poster <poster@example.invalid>',
+        'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
+        'message-id': `<${id()}>`,
+        ':bytes': '700000',
+      },
+      GROUP,
+      n
+    )!
+  )
+}
+
+async function dropSignature(tool: string, group = GROUP) {
+  await db.table('upload_signatures').insert({ tool, group_name: group, probed: 500, named: 0, dropped: true })
+  GarbageFilter.reset()
+}
+
+/** A random-named release with one stored file, as seeding leaves it. */
+async function storedRelease(release: Record<string, unknown> = {}, file: Record<string, unknown> = {}) {
+  const now = new Date()
+  n++
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `garbage${n}`.padEnd(40, 'g'), name: 'Hs7dKq2LmZ9xWv3RtYp1Qa', source: 'stem', groups: JSON.stringify([GROUP]), first_seen_at: now, last_seen_at: now, ...release })
+    .returning(['id', 'release_key'])
+  const [{ id }] = await FileStore.insert([
+    { filename: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', poster: 'p@ngPost.com', date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: JSON.stringify([GROUP]),
+      segments: [{ number: 1, id: `${n.toString(16).padStart(32, 'a')}@ngPost`, size: 10 }], created_at: now, updated_at: now, ...file },
+  ])
+  return { releaseId: Number(r.id), releaseKey: r.release_key as string, fileId: id }
+}
+
+test.group('Garbage filter (D42)', (group) => {
+  const explore = indexerConfig.garbage.explorePercent
+  group.each.setup(() => {
+    // Nothing explored, so a test's release is dropped whatever its key hashes to.
+    indexerConfig.garbage.explorePercent = 0
+    GarbageFilter.reset()
+    return testUtils.db().withGlobalTransaction()
+  })
+  group.each.teardown(() => {
+    indexerConfig.garbage.explorePercent = explore
+    GarbageFilter.reset()
+  })
+  group.teardown(() => QueueService.closeAll())
+
+  test('posting tools are told apart by their Message-IDs', ({ assert }) => {
+    assert.equal(uploadTool('FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'), 'nyuu')
+    assert.equal(uploadTool('b94268bc702040f7b811d5318f727e4f@ngPost'), 'ngPost')
+    assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM')
+    assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp')
+    assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random')
+    assert.equal(uploadTool('seed-1-12@test'), 'other')
+    assert.equal(uploadTool(null), 'other')
+  })
+
+  test('the explored share is stable per key and about the percentage', ({ assert }) => {
+    assert.equal(explored('some-release', 2), explored('some-release', 2))
+    assert.isFalse(explored('some-release', 0))
+    const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length
+    assert.isAbove(kept, 150)
+    assert.isBelow(kept, 250)
+  })
+
+  test('seeding skips random-named files of a dropped signature, not readable or other ones', async ({ assert }) => {
+    await dropSignature('ngPost')
+    const aggregator = new FileAggregator<IngestSegment>()
+    const files = aggregator
+      .add([...upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2), ...upload('Some.Show.S01E01.1080p.WEB.part01.rar', 2), ...upload('Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar', 2, 'nyuu')])
+      .map(toSeedFile)
+    assert.lengthOf(files, 3)
+    const result = await SeedService.store(files)
+    assert.include(result, { files: 2, dropped: 1 })
+    assert.isNull(await db.from('files').where('filename', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
+    assert.isNull(await db.from('releases').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa').first())
+    assert.isNotNull(await db.from('files').where('filename', 'Some.Show.S01E01.1080p.WEB.part01.rar').first())
+    assert.isNotNull(await db.from('files').where('filename', 'Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar').first())
+  })
+
+  test('ingest skips them too, and a release with a readable file is kept whole', async ({ assert }) => {
+    await dropSignature('ngPost')
+    await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2))
+    assert.isNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
+
+    // Off: everything is stored.
+    indexerConfig.garbage.enabled = false
+    try {
+      await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar', 2))
+    } finally {
+      indexerConfig.garbage.enabled = true
+    }
+    assert.isNotNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar').first())
+  })
+
+  test('a release dropped after its probe is not stored again', async ({ assert }) => {
+    const aggregator = new FileAggregator<IngestSegment>()
+    const files = aggregator.add(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2)).map(toSeedFile)
+    await db.table('dropped_releases').insert({ release_key: files[0].release!.key, reason: 'encrypted' })
+    assert.include(await SeedService.store(files), { files: 0, dropped: 1 })
+  })
+
+  test('after its probe, an encrypted random-named release without name or password is deleted', async ({ assert }) => {
+    const garbage = await storedRelease({ encrypted: true })
+    assert.equal(await GarbageFilter.afterProbe(garbage.fileId, 'rar5'), 'encrypted')
+    assert.isNull(await db.from('files').where('id', garbage.fileId).first())
+    assert.isNull(await db.from('releases').where('id', garbage.releaseId).first())
+    assert.equal((await db.from('dropped_releases').where('release_key', garbage.releaseKey).first()).reason, 'encrypted')
+
+    const container = await storedRelease({ container: 'rclone', encrypted: true })
+    assert.equal(await GarbageFilter.afterProbe(container.fileId, 'rclone'), 'rclone')
+    const missing = await storedRelease()
+    assert.equal(await GarbageFilter.afterProbe(missing.fileId, 'missing'), 'missing')
+  })
+
+  test('kept after the probe: a password, a real name, other probes waiting, nothing unreadable found', async ({ assert }) => {
+    const password = await storedRelease({ encrypted: true, password: 'secret', password_source: 'spot' })
+    assert.isNull(await GarbageFilter.afterProbe(password.fileId, 'rar5'))
+    const named = await storedRelease({ encrypted: true, name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
+    assert.isNull(await GarbageFilter.afterProbe(named.fileId, 'rar5'))
+
+    const waiting = await storedRelease({ encrypted: true })
+    const now = new Date()
+    await db.table('binaries').insert({ file_key: `waiting${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.par2', total_parts: 1, groups: '[]',
+      first_seen_at: now, last_seen_at: now, release_id: waiting.releaseId, awaiting_inspection: true })
+    assert.isNull(await GarbageFilter.afterProbe(waiting.fileId, 'rar5'))
+
+    const readable = await storedRelease()
+    assert.isNull(await GarbageFilter.afterProbe(readable.fileId, 'rar5'))
+    assert.isNotNull(await db.from('releases').where('id', readable.releaseId).first())
+  })
+
+  test('learns dropped signatures from probed releases, and keeps a verdict without enough probes', async ({ assert }) => {
+    const { minProbes } = indexerConfig.garbage
+    const now = new Date()
+    const releases = await db
+      .table('releases')
+      .insert(Array.from({ length: minProbes }, (_, i) => ({ release_key: `learn${i}`.padEnd(40, 'l'), name: `Hs7dKq2LmZ9xWv3RtYp${i}`, groups: '[]', first_seen_at: now, last_seen_at: now })))
+      .returning('id')
+    await db.table('signature_probes').insert(releases.map((r) => ({ release_id: r.id, tool: 'ngPost', group_name: GROUP })))
+    // Five probes of another signature: too few to decide.
+    await db.table('signature_probes').insert(releases.slice(0, 5).map((r, i) => ({ release_id: Number(r.id) + 1_000_000_000 + i, tool: 'nyuu', group_name: GROUP })))
+
+    const learned = await GarbageFilter.learn()
+    assert.include(learned.changed.join('\n'), `ngPost ${GROUP}: dropped`)
+    const rows = await db.from('upload_signatures').where('group_name', GROUP).orderBy('tool')
+    assert.deepEqual(rows.map((r) => [r.tool, r.dropped, r.probed]), [['ngPost', true, minProbes], ['nyuu', false, 5]])
+
+    // One real name in a hundred is above the 0.5% limit: kept again.
+    await db.from('releases').where('id', releases[0].id).update({ name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
+    await GarbageFilter.learn()
+    assert.isFalse((await db.from('upload_signatures').where({ tool: 'ngPost', group_name: GROUP }).first()).dropped)
+  })
+
+  test('a stored file of a dropped signature is not probed, and stops waiting for it', async ({ assert }) => {
+    const stored = await storedRelease()
+    const now = new Date()
+    await db.table('binaries').insert({ file_key: `skip${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', total_parts: 1, groups: '[]',
+      first_seen_at: now, last_seen_at: now, release_id: stored.releaseId, file_id: stored.fileId, awaiting_inspection: true })
+    assert.isNotNull(await ContentService.prepare(stored.fileId))
+    await dropSignature('ngPost')
+    assert.isNull(await ContentService.prepare(stored.fileId))
+    assert.isNull((await db.from('binaries').where('file_id', stored.fileId).first()).awaiting_inspection)
+  })
+})

tests/functional/inspection_queue.spec.ts

@@ -0,0 +1,36 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import QueueService from '#services/QueueService'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
+
+test.group('Inspection queue', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.teardown(() => QueueService.closeAll())
+
+  test('the backlog stays in Postgres; BullMQ gets what fits, in priority order', async ({ assert }) => {
+    const queue = QueueService.collectionQueue
+    await queue.drain()
+    await InspectionQueue.add([9_000_001, 9_000_002], INSPECTION_PRIORITY.low)
+    await InspectionQueue.add([9_000_003])
+    await InspectionQueue.add([9_000_004], INSPECTION_PRIORITY.targeted)
+    // Queued again with a higher priority: one row, the higher priority.
+    await InspectionQueue.add([9_000_001], INSPECTION_PRIORITY.targeted)
+    assert.equal(await InspectionQueue.size(), 4)
+
+    const max = InspectionQueue.feedMax
+    ;(InspectionQueue as any).feedMax = 2
+    try {
+      assert.equal(await InspectionQueue.feed(), 2)
+      const waiting = (await queue.getWaiting()).map((job) => job.data.fileId).sort()
+      assert.deepEqual(waiting, [9_000_001, 9_000_004])
+      // Full: nothing more until the workers take some.
+      assert.equal(await InspectionQueue.feed(), 0)
+      const left = (await db.from('inspection_queue').select('file_id')).map((r) => Number(r.file_id)).sort()
+      assert.deepEqual(left, [9_000_002, 9_000_003])
+    } finally {
+      ;(InspectionQueue as any).feedMax = max
+      await queue.drain()
+    }
+  })
+})

tests/functional/release_complete.spec.ts

@@ -0,0 +1,94 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import ReleaseCompleteness from '#services/ReleaseCompleteness'
+
+/** A release with `n` files (each with a binary of size 100000 + i), named f<i>.bin. */
+async function release(n: number) {
+  const now = new Date()
+  const [row] = await db
+    .table('releases')
+    .insert({ release_key: `complete-${n}`.padEnd(40, '0'), name: `Complete.Test.${n}`, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+    .returning('id')
+  const releaseId = Number(row.id)
+  await db.rawQuery(
+    `WITH f AS (
+       INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
+       SELECT 'f' || i || '.bin', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'complete-' || ?::int || '-' || i, now(), now()
+       FROM generate_series(1, ?::int) i
+       RETURNING id, filename)
+     INSERT INTO binaries (file_key, poster, subject, name, total_parts, groups, first_seen_at, last_seen_at, size, file_id, release_id)
+     SELECT 'complete-b-' || ?::int || '-' || f.id, 'x', 'x', f.filename, 1, '[]', now(), now(), 100000 + substr(f.filename, 2, length(f.filename) - 5)::int, f.id, ?::bigint
+     FROM f`,
+    [releaseId, n, n, n, releaseId]
+  )
+  return releaseId
+}
+
+/** The par2 listing of `count` files f1..f<count> (sizes as in release()), in one INSERT, as Par2Service.attach does. */
+async function listing(releaseId: number, count: number, offset = 0) {
+  await db.rawQuery(
+    `INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
+     SELECT ?::bigint, md5(i::text), 'f' || i || '.bin', 100000 + i, md5(i::text), 'set' FROM generate_series(1 + ?::int, ?::int + ?::int) i`,
+    [releaseId, offset, count, offset]
+  )
+  // Triggers only note the release; gc:scheduler's sweep recomputes it.
+  await ReleaseCompleteness.sweep()
+  return Boolean((await db.from('releases').where('id', releaseId).first()).complete)
+}
+
+test.group('release completeness', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('file inserts keep counts, size, extensions and search text current, once per statement', async ({ assert }) => {
+    const id = await release(3)
+    const row = await db.from('releases').where('id', id).first()
+    assert.equal(Number(row.file_count), 3)
+    assert.deepEqual(row.extensions, ['bin'])
+    // No par2 listing, no file total, not complete: its completeness can't change, so it isn't noted.
+    const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
+    assert.equal(Number(count), 0)
+  })
+
+  test('a release with a par2 listing is noted once per insert, and complete by file names alone', async ({ assert }) => {
+    const now = new Date()
+    const [row] = await db
+      .table('releases')
+      .insert({ release_key: 'names-only'.padEnd(40, '0'), name: 'Names.Only', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+      .returning('id')
+    const id = Number(row.id)
+    await db.rawQuery(
+      `INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
+       SELECT ?::bigint, md5(i::text), 'n' || i || '.rar', 100000 + i, md5(i::text), 'set' FROM generate_series(1, 3) i`,
+      [id]
+    )
+    await ReleaseCompleteness.sweep()
+    // Seeded files: no binaries, matched by name only.
+    await db.rawQuery(
+      `INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
+       SELECT 'n' || i || '.rar', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'names-only-' || i, now(), now() FROM generate_series(1, 3) i`,
+      [id]
+    )
+    const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
+    assert.equal(Number(count), 1)
+    await ReleaseCompleteness.sweep()
+    assert.isTrue(Boolean((await db.from('releases').where('id', id).first()).complete))
+  })
+
+  test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {
+    const id = await release(3)
+    assert.isTrue(await listing(id, 3))
+    const other = await release(2)
+    // f3 is listed but missing.
+    assert.isFalse(await listing(other, 3))
+  })
+
+  test('a par2 listing of thousands of files is checked in one go', async ({ assert }) => {
+    const id = await release(3000)
+    await ReleaseCompleteness.sweep()
+    const started = Date.now()
+    assert.isTrue(await listing(id, 3000))
+    // Was cubic: minutes for a few thousand rows.
+    assert.isBelow(Date.now() - started, 5000)
+  }).timeout(60_000)
+})

tests/functional/seed.spec.ts

@@ -5,8 +5,8 @@ import { segmentFromHeader, type IngestSegment } from '#services/IngestService'
 import SeedService from '#services/SeedService'
 import QueueService from '#services/QueueService'
 import { FileAggregator } from '#lib/aggregate/index'
-import { toSeedFile } from '#lib/seed/wire'
-import { segmentsOf } from '#services/FileStore'
+import { toFragment, toSeedFile } from '#lib/seed/wire'
+import { insertLimits, segmentsOf } from '#services/FileStore'
 
 function header(subject: string, n: number) {
   return {
@@ -37,12 +37,51 @@ test.group('SeedService', (group) => {
     assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
     const release = await db.from('releases').where('id', stored.release_id).first()
     assert.equal(release.name, segments[0].release!.name) // named as live indexing would
-    const binary = await db.from('binaries').where('file_key', stored.file_key).first()
-    assert.equal(Number(binary.file_id), Number(stored.id))
-    assert.isNotNull(binary.completed_at)
+    // A readable name isn't inspected: no binary (LOG.md 2026-10-06 (11)).
+    assert.isNull(await db.from('binaries').where('file_key', stored.file_key).first())
 
     // The same files again (a crosspost, a second run): nothing new.
     const again = await SeedService.store(files)
     assert.include(again, { files: 0, existing: 2 })
   })
+
+  test('an obfuscated file keeps a binary until inspection has fingerprinted it', async ({ assert }) => {
+    const aggregator = new FileAggregator<IngestSegment>()
+    const subject = 'Xq8r2KdLm3Zw91PqA7zz [1/1] - "Xq8r2KdLm3Zw91PqA7zz.bin" yEnc (%d/2)'
+    const segments = [1, 2].map((n) => segmentFromHeader(header(subject.replace('%d', String(n)), n), 'alt.binaries.test', n)!)
+    const files = aggregator.add(segments).map(toSeedFile)
+    const result = await SeedService.store(files)
+    assert.include(result, { files: 1, inspections: 1 })
+    const binary = await db.from('binaries').where('file_key', files[0].fileKey).first()
+    assert.isTrue(binary.awaiting_inspection)
+    assert.isNotNull(binary.file_id)
+  })
+
+  test('stores a batch in several INSERTs', async ({ assert }) => {
+    // Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
+    const rows = insertLimits.rows
+    insertLimits.rows = 50
+    try {
+      const aggregator = new FileAggregator<IngestSegment>()
+      const segments = Array.from({ length: 120 }, (_, i) =>
+        segmentFromHeader(header(`Dense.Upload - [${i + 1}/120] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
+      )
+      const files = aggregator.add(segments).map(toSeedFile)
+      assert.include(await SeedService.store(files), { files: 120 })
+      const [{ count }] = await db.from('files').whereLike('filename', 'dense.%').count('* as count')
+      assert.equal(Number(count), 120)
+    } finally {
+      insertLimits.rows = rows
+    }
+  })
+
+  test('fragments are kept near the range edges only', async ({ assert }) => {
+    const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
+    const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!
+    const middle = segmentFromHeader(header(subject('middle.bin'), 2), 'alt.binaries.test', 1_500_000)!
+    const kept = await SeedService.storeFragments([toFragment(edge), toFragment(middle)], { start: 1_000_000, end: 2_999_999 })
+    assert.equal(kept, 1)
+    assert.isOk(await db.from('binaries').where('name', 'edge.bin').first())
+    assert.isNotOk(await db.from('binaries').where('name', 'middle.bin').first())
+  })
 })

tests/functional/seed_api.spec.ts

@@ -11,6 +11,9 @@ import { segmentFromHeader } from '#lib/ingest/segment'
 import { FileAggregator } from '#lib/aggregate/index'
 import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
 import type { IngestSegment } from '#lib/ingest/segment'
+import FileStore from '#services/FileStore'
+import RemoteContentService from '#services/RemoteContentService'
+import { Job } from 'bullmq'
 
 function header(subject: string, n: number) {
   return { 'subject': subject, 'from': 'api seeder <api@example.invalid>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<api-${n}-${subject.length}@test>`, ':bytes': '500000' }
@@ -25,7 +28,10 @@ function batch(next: number): SeedBatch {
 
 test.group('Seed API', (group) => {
   group.each.setup(() => testUtils.db().withGlobalTransaction())
-  group.teardown(() => QueueService.closeAll())
+  group.teardown(async () => {
+    await RemoteContentService.close()
+    await QueueService.closeAll()
+  })
 
   test('a node leases a range, posts a gzip batch, and finishes it', async ({ client, assert }) => {
     const token = await SeedOrchestrator.issueToken('test-node')
@@ -48,9 +54,8 @@ test.group('Seed API', (group) => {
     assert.include(((await posted.json()) as any).stored, { files: 1 })
     const file = await db.from('files').where('filename', 'Api.Test.2026.mkv').first()
     assert.equal(file.parts, 2)
-    // The lone .nfo segment took the regular path (staged or stored as a binary).
-    const fragment = (await db.from('staged_segments').count('* as n').first()).n + (await db.from('binaries').where('name', 'Api.Test.2026.nfo').count('* as n').first()).n
-    assert.isAbove(Number(fragment), 0)
+    // The lone .nfo segment sits near the range's edge: kept as a binary (not staged) for the neighbouring range.
+    assert.isOk(await db.from('binaries').where('name', 'Api.Test.2026.nfo').first())
 
     const row = await db.from('seed_ranges').where('id', range.id).first()
     assert.equal(Number(row.next_id), 51)
@@ -128,4 +133,133 @@ test.group('Seed API', (group) => {
       indexerConfig.seed.nntpConnections = connections
     }
   })
+
+  test('ranges of a higher priority are leased first', async ({ client, assert }) => {
+    const token = await SeedOrchestrator.issueToken('priority-node')
+    await db.table('seed_ranges').insert([
+      { group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 },
+      { group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101, priority: 1 },
+    ])
+    const leased = await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })
+    assert.include(leased.body().range, { start: 101, end: 200 })
+  })
+
+  test('priorities take turns: half the leases go to the higher priority', async ({ client, assert }) => {
+    const token = await SeedOrchestrator.issueToken('turns-node')
+    const auth = { authorization: `Bearer ${token}` }
+    await db.table('seed_ranges').insert([
+      { group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 },
+      { group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101 },
+      { group_name: 'alt.binaries.test', start_id: 1001, end_id: 1100, next_id: 1001, priority: 1 },
+      { group_name: 'alt.binaries.test', start_id: 1101, end_id: 1200, next_id: 1101, priority: 1 },
+    ])
+    const starts: number[] = []
+    for (let i = 0; i < 4; i++) starts.push((await client.post('/api/v1/seed/lease').headers(auth)).body().range.start)
+    assert.deepEqual(starts, [1001, 1, 1101, 101])
+  })
+
+  test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => {
+    await QueueService.collectionQueue.drain()
+    await QueueService.archiveWalkQueue.drain()
+    const token = await SeedOrchestrator.issueToken('inspect-node')
+    const auth = { authorization: `Bearer ${token}` }
+    const now = new Date()
+    const [release] = await db
+      .table('releases')
+      .insert({ release_key: 'i'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7zz', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+      .returning('id')
+    const [file] = await FileStore.insert([
+      {
+        filename: 'Xq8r2KdLm3Zw91PqA7zz.part01.rar',
+        file_key: 'inspect-node-file'.padEnd(40, '0'),
+        poster: 'x',
+        date: now.getTime(),
+        parts: 2,
+        groups: '[]',
+        segments: [
+          { number: 1, id: 'one@test', size: 100 },
+          { number: 2, id: 'two@test', size: 100 },
+        ],
+        release_id: release.id,
+        created_at: now,
+        updated_at: now,
+      },
+    ])
+    await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
+
+    const leased = await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)
+    leased.assertStatus(200)
+    const job = leased.body().jobs.find((j: any) => j.task.kind === 'inspect' && j.task.fileId === file.id)
+    assert.deepEqual(job.task.segments.map((s: any) => s.id), ['one@test', 'two@test'])
+    // Locked: nobody else gets it meanwhile.
+    ;(await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)).assertStatus(204)
+
+    // What probeInspection found in the first segment; posted as a JSON string, empty strings and all.
+    const result = {
+      size: 15_728_640,
+      contentType: null,
+      md5_16k: null,
+      yencName: '',
+      archive: {
+        format: 'rar5', volume: true, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, next: 120,
+        entries: [{ name: 'Real.Release.Name.2026.1080p-GRP/real.release.name.mkv', size: 5_000_000_000, packedSize: 15_700_000, directory: false, encrypted: true, splitBefore: false, splitAfter: true, stored: false, dataOffset: 100 }],
+      },
+    }
+    const done = { queue: job.queue, jobId: job.jobId, token: job.token, result: JSON.stringify(result) }
+    ;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(200)
+
+    const row = await db.from('releases').where('id', release.id).first()
+    assert.equal(row.name, 'Real.Release.Name.2026.1080p-GRP')
+    assert.isTrue(row.encrypted)
+    assert.lengthOf(row.contents, 1)
+    assert.isOk((await db.from('file_details').where('file_id', file.id).first())?.archive)
+    assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
+    // Too late (or twice): the lock is gone.
+    ;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(409)
+  })
+
+  test('a node gives an inspection job back, or reports it failed', async ({ client, assert }) => {
+    await QueueService.collectionQueue.drain()
+    await QueueService.archiveWalkQueue.drain()
+    const token = await SeedOrchestrator.issueToken('inspect-node-2')
+    const auth = { authorization: `Bearer ${token}` }
+    const now = new Date()
+    const [file] = await FileStore.insert([
+      { filename: 'Zk3m9QpL2xW8rT5vN1bY.bin', file_key: 'inspect-node-file-2'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'x@test', size: 1 }], created_at: now, updated_at: now },
+    ])
+    await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
+    const lease = async () => (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
+
+    const first = await lease()
+    ;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200)
+    const again = await lease()
+    assert.equal(again.jobId, first.jobId)
+    ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: 'NNTP BODY: no answer within 120 s' }).headers(auth)).assertStatus(200)
+    // Retried later (backoff), like a local failure.
+    assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState())
+  })
+
+  test('a missing first article reported by a node is recorded, not retried', async ({ client, assert }) => {
+    await QueueService.collectionQueue.drain()
+    await QueueService.archiveWalkQueue.drain()
+    const token = await SeedOrchestrator.issueToken('inspect-node-3')
+    const auth = { authorization: `Bearer ${token}` }
+    const now = new Date()
+    const [file] = await FileStore.insert([
+      { filename: 'Wq7n3LpX9vT2rK8mZ5cYbA.bin', file_key: 'inspect-node-file-3'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'gone@test', size: 1 }], created_at: now, updated_at: now },
+    ])
+    await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
+    const job = (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
+    const error = 'Seed API /api/v1/seed/articles?id=gone%40test: 410 {"error":"No such article"}'
+    ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: job.queue, jobId: job.jobId, token: job.token, error }).headers(auth)).assertStatus(200)
+    assert.equal((await db.from('file_details').where('file_id', file.id).first())?.content_type, 'missing')
+    assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
+  })
+
+  test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => {
+    const token = await SeedOrchestrator.issueToken('late-node')
+    const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id')
+    ;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204)
+    assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'done')
+  })
 })

tests/functional/spot_listing.spec.ts

@@ -2,7 +2,7 @@ import { test } from '@japa/runner'
 import { createHash } from 'node:crypto'
 import db from '@adonisjs/lucid/services/db'
 import testUtils from '@adonisjs/core/services/test_utils'
-import SpotService, { isDummyNzb } from '#services/SpotService'
+import SpotService, { isDummyNzb, NzbUnreadable } from '#services/SpotService'
 import SearchService from '#services/SearchService'
 
 test.group('SpotService listings', (group) => {
@@ -107,3 +107,87 @@ test.group('SpotService passwords', (group) => {
     assert.equal(passwordState(null, false), 'none')
   })
 })
+
+test.group('SpotService reading listings', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  const nzb = (password: string | null, bytes = 700_000_000) => `<?xml version="1.0" encoding="UTF-8"?>
+<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb"><head>${password ? `<meta type="password">${password}</meta>` : ''}</head>
+<file poster="p" date="1700000000" subject="&quot;a.part1.rar&quot; yEnc (1/1)"><groups><group>alt.binaries.test</group></groups>
+<segments><segment bytes="${bytes}" number="1">a1@x</segment></segments></file></nzb>`
+
+  /** A listed spot, as worker:spots leaves an old one, with its article and NZB as given. */
+  async function listedSpot(id: string, read: { description?: string; nzb?: string | null; error?: string; nzbError?: Error }) {
+    await db.table('spots').insert({
+      message_id: id, group_name: 'free.pt', article_number: 1, title: `Listed ${id}`, poster: 'Bart', key_id: 7, size: 700_000_000,
+      posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: new Date(), status: 'pending',
+    })
+    const [claimed] = await SpotService.claim(10)
+    await SpotService.list([claimed])
+    const service = SpotService as any
+    service.readHead = async () => {
+      if (read.error) throw new Error(read.error)
+      return { header: { size: 700_000_000 }, xml: { description: read.description ?? '', nzbSegments: ['n@x'] } }
+    }
+    service.readNzb = async () => {
+      if (read.nzbError) throw read.nzbError
+      return read.nzb ?? null
+    }
+  }
+
+  const restore = () => {
+    delete (SpotService as any).readHead
+    delete (SpotService as any).readNzb
+  }
+  group.each.teardown(restore)
+
+  const passwordOf = async (id: string) => {
+    const spot = await db.from('spots').where('message_id', id).first()
+    const release = await db.from('releases').where('id', spot.release_id).first()
+    return { spot, password: release.password, source: release.password_source }
+  }
+
+  test("reads the NZB's password and the details, without importing files", async ({ assert }) => {
+    await listedSpot('l1@spot.net', { description: 'Plot', nzb: nzb('fromnzb1') })
+    assert.deepEqual(await SpotService.readListings(10, 2), { 'nzb-password': 1 })
+    const { spot, password, source } = await passwordOf('l1@spot.net')
+    assert.deepEqual([password, source, spot.status, spot.description], ['fromnzb1', 'nzb', 'listed', 'Plot'])
+    assert.isNotNull(spot.nzb_read_at)
+    assert.equal((await db.from('releases').where('id', spot.release_id).first()).file_count, 0)
+    // Read once: nothing left.
+    assert.deepEqual(await SpotService.readListings(10, 2), {})
+  })
+
+  test("the spot's text when the NZB has none; the NZB's own password wins over it", async ({ assert }) => {
+    await listedSpot('l2@spot.net', { description: 'Wachtwoord: fromtext2', nzb: nzb(null) })
+    assert.deepEqual(await SpotService.readListings(10, 1), { 'spot-password': 1 })
+    assert.include(await passwordOf('l2@spot.net'), { password: 'fromtext2', source: 'spot' })
+
+    await listedSpot('l3@spot.net', { description: 'Wachtwoord: fromtext3', nzb: nzb('fromnzb3') })
+    await db.from('releases').where('id', (await db.from('spots').where('message_id', 'l3@spot.net').first()).release_id).update({ password: 'fromtext3', password_source: 'spot' })
+    assert.deepEqual(await SpotService.readListings(10, 1), { 'nzb-password': 1 })
+    assert.include(await passwordOf('l3@spot.net'), { password: 'fromnzb3', source: 'nzb' })
+  })
+
+  test('a placeholder NZB hides the listing; a lost connection gives the spot back, a gone article does not', async ({ assert }) => {
+    await listedSpot('l4@spot.net', { nzb: nzb(null, 1500) })
+    assert.deepEqual(await SpotService.readListings(10, 1), { dummy: 1 })
+    assert.equal((await db.from('spots').where('message_id', 'l4@spot.net').first()).status, 'dummy')
+
+    await listedSpot('l5@spot.net', { error: 'Connection reset' })
+    assert.deepEqual(await SpotService.readListings(10, 1), { retry: 1 })
+    assert.isNull((await db.from('spots').where('message_id', 'l5@spot.net').first()).nzb_read_at)
+
+    await db.from('spots').where('message_id', 'l5@spot.net').delete()
+    await listedSpot('l6@spot.net', { error: '430 No Such Article' })
+    assert.deepEqual(await SpotService.readListings(10, 1), { gone: 1 })
+    assert.isNotNull((await db.from('spots').where('message_id', 'l6@spot.net').first()).nzb_read_at)
+
+    // A corrupt NZB is a result, not something to read again.
+    await listedSpot('l7@spot.net', { nzbError: new NzbUnreadable('invalid code lengths set') })
+    assert.deepEqual(await SpotService.readListings(10, 1), { 'nzb-unreadable': 1 })
+    const l7 = await db.from('spots').where('message_id', 'l7@spot.net').first()
+    assert.isNotNull(l7.nzb_read_at)
+    assert.equal(l7.error, 'NZB unreadable: invalid code lengths set')
+  })
+})

tests/functional/web_api.spec.ts

@@ -1,4 +1,5 @@
 import { test } from '@japa/runner'
+import ReleaseCompleteness from '#services/ReleaseCompleteness'
 import db from '@adonisjs/lucid/services/db'
 import FileStore from '#services/FileStore'
 import testUtils from '@adonisjs/core/services/test_utils'
@@ -89,9 +90,12 @@ test.group('Web API', (group) => {
     response.assertBodyContains({ sources: [] })
   })
 
-  test('completeness follows files and par2 lists through triggers', async ({ assert }) => {
+  test('completeness follows files and par2 lists (triggers note releases, the sweep recomputes)', async ({ assert }) => {
     const now = new Date()
-    const complete = async (id: number) => (await db.from('releases').where('id', id).first()).complete
+    const complete = async (id: number) => {
+      await ReleaseCompleteness.sweep()
+      return (await db.from('releases').where('id', id).first()).complete
+    }
     const file = (releaseId: number, name: string) => ({
       filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
       groups: '[]', segments: [], created_at: now, updated_at: now,
@@ -131,3 +135,11 @@ test.group('Web API', (group) => {
     assert.isTrue(row.par2Only)
   })
 })
+
+test.group('Health check', () => {
+  test('/health answers when the database does', async ({ client }) => {
+    const response = await client.get('/health')
+    response.assertStatus(200)
+    response.assertBody({ ok: true })
+  })
+})

tests/unit/aggregate.spec.ts

@@ -37,3 +37,20 @@ test.group('aggregate / FileAggregator', () => {
     assert.equal(aggregator.openFiles, 0)
   })
 })
+
+test.group('seed / runRange', () => {
+  test('a range already past its end only reports that it is done', async ({ assert }) => {
+    const { runRange } = await import('#lib/seed/runner')
+    const batches: any[] = []
+    await runRange({
+      client: { group: async () => assert.fail('no GROUP'), xover: async () => assert.fail('no XOVER') },
+      group: 'a.b.test',
+      start: 101,
+      end: 100,
+      sink: async (batch) => {
+        batches.push(batch)
+      },
+    })
+    assert.deepEqual(batches.map((b) => [b.next, b.headers, b.files.length]), [[101, 0, 0]])
+  })
+})

tests/unit/probe.spec.ts

@@ -0,0 +1,86 @@
+import { test } from '@japa/runner'
+import { crc32Hex } from '#lib/yenc/index'
+import { probeInspection, type InspectTask } from '#lib/content/probe'
+
+/** One yEnc article of `file`: part `index` of `size` bytes (escaping '.' and whitespace too, so no dot-stuffing). */
+function yencArticle(name: string, file: Buffer, index: number, size: number) {
+  const data = file.subarray(index * size, (index + 1) * size)
+  const total = Math.ceil(file.length / size)
+  const out: number[] = []
+  for (const byte of data) {
+    const c = (byte + 42) & 0xff
+    if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
+    else out.push(c)
+  }
+  const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
+  const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
+  return Buffer.from(head + Buffer.from(out).toString('latin1') + tail, 'latin1')
+}
+
+/** A posted file as articles, and its inspection task. */
+function posted(name: string, file: Buffer, size = 400) {
+  const articles = new Map<string, Buffer>()
+  const segments = []
+  for (let i = 0; i * size < file.length; i++) {
+    const id = `${name}-${i}@test`
+    articles.set(id, yencArticle(name, file, i, size))
+    segments.push({ number: i + 1, id, size: size + 100 })
+  }
+  const fetched: string[] = []
+  const fetch = async (id: string) => {
+    fetched.push(id)
+    const body = articles.get(id)
+    if (!body) throw new Error(`430 no such article ${id}`)
+    return body
+  }
+  const task: InspectTask = { kind: 'inspect', fileId: 1, name, segments, sevenZip: { volumes: 1, last: segments } }
+  return { task, fetch, fetched }
+}
+
+// `7z a -pSecret -mhe=on`: an encrypted header (tests/unit/sevenzip.spec.ts).
+const HEADER_ENCRYPTED = Buffer.from(
+  'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
+  'base64'
+)
+
+test.group('content probe', () => {
+  test('a first segment gives size, fingerprint and the yEnc name', async ({ assert }) => {
+    const file = Buffer.alloc(20_000, 7)
+    const { task, fetch, fetched } = posted('Some.Name.bin', file, 17_000)
+    const probe = await probeInspection(task, fetch)
+    assert.include(probe, { size: 20_000, contentType: null, yencName: 'Some.Name.bin' })
+    assert.match(probe.md5_16k!, /^[0-9a-f]{32}$/)
+    assert.deepEqual(fetched, ['Some.Name.bin-0@test'])
+  })
+
+  test('an NZB is read whole', async ({ assert }) => {
+    const xml = `<?xml version="1.0"?>\n<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${'<file subject="x"/>'.repeat(60)}</nzb>`
+    const { task, fetch, fetched } = posted('x.nzb', Buffer.from(xml))
+    const probe = await probeInspection(task, fetch)
+    assert.equal(probe.contentType, 'nzb')
+    assert.equal(probe.text, xml)
+    // Every segment once: the first, decoded to sniff, is reused for the whole.
+    assert.deepEqual(fetched, task.segments.map((s) => s.id))
+  })
+
+  test('a 7z archive with encrypted headers is known from its tail, without bsdtar', async ({ assert }) => {
+    const { task, fetch } = posted('a.7z', HEADER_ENCRYPTED, 100)
+    const probe = await probeInspection(task, fetch)
+    assert.equal(probe.contentType, '7z')
+    assert.include(probe.archive!, { format: '7z', headersEncrypted: true })
+    assert.lengthOf(probe.archive!.entries, 0)
+  })
+
+  test('a missing first article is a result, not a failure (no retries)', async ({ assert }) => {
+    const { task, fetch } = posted('gone.bin', Buffer.alloc(10, 1))
+    task.segments[0].id = 'missing@test'
+    assert.include(await probeInspection(task, fetch), { contentType: 'missing' })
+    // Through the seed relay: 410.
+    const relayed = await probeInspection(task, async () => {
+      throw new Error('Seed API /api/v1/seed/articles?id=x: 410 {"error":"No such article"}')
+    })
+    assert.equal(relayed.contentType, 'missing')
+    // Anything else still fails (and is retried).
+    await assert.rejects(() => probeInspection(task, async () => { throw new Error('NNTP BODY: no answer within 120 s') }), /no answer/)
+  })
+})

tests/unit/rar.spec.ts

@@ -1,6 +1,7 @@
 import { test } from '@japa/runner'
 import { crc32 } from 'node:zlib'
-import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, type RarEntry } from '#lib/rar/index'
+import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, walkRar, type RarEntry, type VolumeReader } from '#lib/rar/index'
+import { crc32Hex } from '#lib/yenc/index'
 
 /** Entries without the extraction fields, for comparing listings. */
 const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest)
@@ -210,6 +211,19 @@ test.group('rar / release naming from contents', () => {
     assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91Pq/x.mkv'])))
   })
 
+  test('hidden entries (a decoy) name nothing', async ({ assert }) => {
+    const { releaseNameFromContents, isHidden } = await import('#services/ArchiveService')
+    const decoy = '.1917-2019-2160p_MA_WEB-DL_Hybrid_H265_DV_HDR_DDP_Atmos_5.1_English-HONE-.mkv'
+    const listing = info([decoy, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX', `${decoy}/${decoy}.19OMnL`, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX/boston.blue.s01e02.rar'])
+    listing.entries[0].directory = true
+    listing.entries[1].directory = true
+    listing.entries[2].size = 22_093_429_730
+    assert.equal(releaseNameFromContents(listing), 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX')
+    assert.isTrue(isHidden(`${decoy}/x.mkv`))
+    assert.isTrue(isHidden('dir/.hidden.mkv'))
+    assert.isFalse(isHidden('Some.Release/movie.mkv'))
+  })
+
   test('recognises obfuscated names', async ({ assert }) => {
     const { looksRandom } = await import('#services/ArchiveService')
     assert.isTrue(looksRandom('CGBWgdhlSCAOAW62gLWRRM'))
@@ -228,3 +242,135 @@ test.group('rar / looksRandom capitals', () => {
     assert.isFalse(looksRandom('TheQuickBrownFoxJumps'))
   })
 })
+
+// --- Archive walks (D40) --------------------------------------------------------
+
+/** Volumes split into segments of `segmentSize` bytes; counts the segments read. */
+function memoryReader(volumes: Buffer[], segmentSize: number) {
+  const read = new Set<string>()
+  const reader: VolumeReader & { segments: Set<string> } = {
+    segments: read,
+    volumes: volumes.length,
+    size: (v) => volumes[v].length,
+    async read(v, offset, more) {
+      const index = Math.floor(offset / segmentSize)
+      if (offset >= volumes[v].length) return null
+      read.add(`${v}:${index}`)
+      if (more) read.add(`${v}:${index + 1}`)
+      return volumes[v].subarray(offset, (index + (more ? 2 : 1)) * segmentSize)
+    },
+  }
+  return reader
+}
+
+function rar5Set() {
+  const main = (n: number) => rar5Block(1, 0, n === 0 ? vint(0x0001) : Buffer.concat([vint(0x0003), vint(n)]))
+  const end = rar5Block(5, 0, vint(0x0001))
+  const SPLIT_BEFORE = 0x0008
+  const SPLIT_AFTER = 0x0010
+  return [
+    Buffer.concat([RAR5, main(0), rar5File('Book/01.mp3', 3000, { packed: 3000 }), rar5File('Book/02.mp3', 7000, { packed: 5000, headerFlags: SPLIT_AFTER }), end]),
+    Buffer.concat([
+      RAR5,
+      main(1),
+      rar5File('Book/02.mp3', 7000, { packed: 2000, headerFlags: SPLIT_BEFORE }),
+      rar5File('Book/03.mp3', 1000, { packed: 1000 }),
+      rar5File('Book/04.mp3', 4500, { packed: 4000, headerFlags: SPLIT_AFTER }),
+      end,
+    ]),
+    Buffer.concat([RAR5, main(2), rar5File('Book/04.mp3', 4500, { packed: 500, headerFlags: SPLIT_BEFORE }), rar5File('Book/05.mp3', 100, { packed: 100 }), rar5Block(5, 0, vint(0))]),
+  ]
+}
+
+test.group('rar / archive walks', () => {
+  test('lists every entry of a RAR5 set, reading a segment per header', async ({ assert }) => {
+    // 700-byte segments: headers get cut off by segment ends.
+    const reader = memoryReader(rar5Set(), 700)
+    const walk = await walkRar(reader)
+    assert.isTrue(walk.complete)
+    assert.equal(walk.format, 'rar5')
+    assert.deepEqual(walk.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3', 'Book/03.mp3', 'Book/04.mp3', 'Book/05.mp3'])
+    assert.equal(walk.entries[1].size, 7000)
+    const total = rar5Set().reduce((sum, v) => sum + Math.ceil(v.length / 700), 0)
+    assert.isBelow(reader.segments.size, total / 2)
+  })
+
+  test('lists a RAR4 set across volumes', async ({ assert }) => {
+    const end = rar4Block(0x7b, 0, Buffer.alloc(0))
+    const volumes = [
+      Buffer.concat([RAR4, rar4Main(0x0001 | 0x0100), rar4File(Buffer.from('a.flac'), { packed: 900, unpacked: 900 }), rar4File(Buffer.from('b.flac'), { packed: 2000, unpacked: 2500, flags: 0x0002 }), end]),
+      Buffer.concat([RAR4, rar4Main(0x0001), rar4File(Buffer.from('b.flac'), { packed: 500, unpacked: 2500, flags: 0x0001 }), rar4File(Buffer.from('c.flac'), { packed: 10, unpacked: 10 }), end]),
+    ]
+    const walk = await walkRar(memoryReader(volumes, 512))
+    assert.isTrue(walk.complete)
+    assert.deepEqual(walk.entries.map((e) => e.name), ['a.flac', 'b.flac', 'c.flac'])
+  })
+
+  test('stops at a missing segment or a broken header, keeping what it found', async ({ assert }) => {
+    const set = rar5Set()
+    const missing = memoryReader(set, 700)
+    const read = missing.read.bind(missing)
+    missing.read = async (v, offset, more) => (v === 1 ? null : read(v, offset, more))
+    const partial = await walkRar(missing)
+    assert.equal(partial.stopped, 'unreadable')
+    assert.deepEqual(partial.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3'])
+
+    const broken = rar5Set()
+    // The header of 03.mp3 in volume 2, after 02.mp3's continuation and its data.
+    const at = broken[1].indexOf(Buffer.from('Book/03.mp3')) - 4
+    broken[1][at] ^= 0xff
+    const invalid = await walkRar(memoryReader(broken, 700))
+    assert.equal(invalid.stopped, 'invalid')
+    assert.lengthOf(invalid.entries, 2)
+  })
+
+  test('finds segments from their yEnc offsets', async ({ assert }) => {
+    const { SegmentReader } = await import('#lib/rar/index')
+    const set = rar5Set()
+    const size = 700
+    const articles = new Map<string, Buffer>()
+    const files = set.map((volume, v) => {
+      const segments = []
+      for (let i = 0; i * size < volume.length; i++) {
+        const id = `v${v}s${i}@test`
+        articles.set(id, yencArticle(`x.part${v + 1}.rar`, volume, i, size))
+        segments.push({ number: i + 1, id, size: 0 })
+      }
+      return segments
+    })
+    const reader = new SegmentReader(files, 100, async (id) => articles.get(id)!)
+    const walk = await walkRar(reader)
+    assert.isTrue(walk.complete)
+    assert.lengthOf(walk.entries, 5)
+    assert.isBelow(reader.fetched, articles.size / 2)
+
+    const capped = new SegmentReader(files, 3, async (id) => articles.get(id)!)
+    const stopped = await walkRar(capped)
+    assert.equal(stopped.stopped, 'unreadable')
+    assert.equal(capped.fetched, 3)
+  })
+})
+
+/** One yEnc article of `file`: part `index` of `size` bytes. */
+function yencArticle(name: string, file: Buffer, index: number, size: number) {
+  const data = file.subarray(index * size, (index + 1) * size)
+  const total = Math.ceil(file.length / size)
+  const out: number[] = []
+  for (const byte of data) {
+    const c = (byte + 42) & 0xff
+    // Escape the critical characters, and '.' and whitespace so no line needs dot-stuffing.
+    if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
+    else out.push(c)
+  }
+  const lines: string[] = []
+  const encoded = Buffer.from(out)
+  for (let i = 0; i < encoded.length; i += 128) {
+    let end = Math.min(encoded.length, i + 128)
+    if (encoded[end - 1] === 0x3d && end < encoded.length) end++
+    lines.push(encoded.subarray(i, end).toString('latin1'))
+    i = end - 128
+  }
+  const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
+  const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
+  return Buffer.from(head + lines.join('\r\n') + tail, 'latin1')
+}

tests/unit/seed_fetch_pool.spec.ts

@@ -0,0 +1,48 @@
+import { test } from '@japa/runner'
+import net from 'node:net'
+import { unzipSync } from 'node:zlib'
+import nntpConfig from '#config/nntp'
+import indexerConfig from '#config/indexer'
+import SeedFetchPool from '#services/SeedFetchPool'
+import { parseOverviews } from '#lib/nntp/xover_client'
+
+test.group('SeedFetchPool', () => {
+  test('a chunk that fails (dead connection, stalled compressed reply) is fetched again uncompressed', async ({ assert }) => {
+    let connections = 0
+    const compressAsked: Record<number, boolean> = {}
+    const server = net.createServer((sock) => {
+      const n = ++connections
+      sock.write('200 welcome\r\n')
+      sock.on('data', (chunk) => {
+        for (const line of chunk.toString().split('\r\n').filter(Boolean)) {
+          if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n')
+          else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n')
+          else if (line.startsWith('XFEATURE')) (compressAsked[n] = true), sock.write('500 no\r\n')
+          else if (line.startsWith('GROUP')) sock.write('211 1 7 7 a.b.test\r\n')
+          // The first connection dies as the request arrives, like one the server dropped while idle.
+          else if (line.startsWith('XOVER') && n === 1) sock.destroy()
+          else if (line.startsWith('XOVER')) sock.write('224 overviews\r\n7\tsubject\tposter\tdate\t<a@b>\t\t10\t1\r\n.\r\n')
+        }
+      })
+    })
+    await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+    const saved = { host: nntpConfig.host, port: nntpConfig.port, secure: nntpConfig.secure, connections: indexerConfig.seed.nntpConnections, compress: indexerConfig.seed.nntpCompress }
+    // The fallback from a compressed pooled connection (the default).
+    indexerConfig.seed.nntpCompress = true
+    Object.assign(nntpConfig, { host: '127.0.0.1', port: (server.address() as net.AddressInfo).port, secure: false })
+    indexerConfig.seed.nntpConnections = 2
+    try {
+      const raw = await SeedFetchPool.fetch('a.b.test', 7, 7)
+      assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [7])
+      assert.equal(connections, 2)
+      // The pooled connection asked for gzip; the retry didn't.
+      assert.deepEqual(compressAsked, { 1: true })
+    } finally {
+      SeedFetchPool.closeIdle()
+      Object.assign(nntpConfig, { host: saved.host, port: saved.port, secure: saved.secure })
+      indexerConfig.seed.nntpConnections = saved.connections
+      indexerConfig.seed.nntpCompress = saved.compress
+      server.close()
+    }
+  })
+})

tests/unit/seed_plan.spec.ts

@@ -0,0 +1,16 @@
+import { test } from '@japa/runner'
+import { cut, uncovered } from '#lib/seed/plan'
+
+test.group('seed plan', () => {
+  test('plans only what earlier ranges leave uncovered', ({ assert }) => {
+    // a.b.ath on 2026-10-06: test ranges around 49.8B and a plan below them; the backfill ends at 58.4B.
+    assert.deepEqual(uncovered(1, 100, [[40, 50], [10, 20], [45, 60]]), [[1, 9], [21, 39], [61, 100]])
+    assert.deepEqual(uncovered(1, 100, []), [[1, 100]])
+    assert.deepEqual(uncovered(1, 100, [[0, 200]]), [])
+    assert.deepEqual(uncovered(10, 20, [[1, 5], [25, 30]]), [[10, 20]])
+  })
+
+  test('cuts gaps into ranges, newest first', ({ assert }) => {
+    assert.deepEqual(cut([[1, 9], [61, 100]], 25), [[76, 100], [61, 75], [1, 9]])
+  })
+})

tests/unit/sevenzip.spec.ts

@@ -1,6 +1,7 @@
 import { test } from '@japa/runner'
 import {
   checkHeader,
+  encodedHeaderEncrypted,
   listingStart,
   parseBsdtarListing,
   parseSevenZipStart,
@@ -19,7 +20,30 @@ const PLAIN = Buffer.from(
   'base64'
 )
 
+// `7z a -pSecret -mhe=on`: an encrypted header; with -mhe=off only the file data is encrypted.
+const HEADER_ENCRYPTED = Buffer.from(
+  'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
+  'base64'
+)
+const DATA_ENCRYPTED = Buffer.from(
+  'N3q8ryccAASBXGg3kQAAAAAAAAAiAAAAAAAAAO4xlWU16N8rTbvoF1dbq3eLPdZiAACBMweuD87ysgwHsMPa919FiphISvi++OI2giqcUPCAdmXzZtkqFSqP0Nr4rWaRH5dR+vNA2jJGIbDAZpbQM/ka9oh27tkqKb/+dHyx1SFHkxbLyWvd7W0qwaDHxSvW2fTpRdJSmi74crF957Qwy+P8dDF0MOYr5AGHAQHJSAAAFwYQAQmAgQAHCwEAASMDAQEFXQAQAAAMgKYKAaUZQIYAAA==',
+  'base64'
+)
+const headerOf = (archive: Buffer) => {
+  const start = parseSevenZipStart(archive)!
+  return archive.subarray(start.headerStart, start.archiveSize)
+}
+
 test.group('sevenzip', () => {
+  test('tells encrypted headers from the encoded header alone', ({ assert }) => {
+    assert.isTrue(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED)))
+    assert.isFalse(encodedHeaderEncrypted(headerOf(DATA_ENCRYPTED)))
+    assert.isFalse(encodedHeaderEncrypted(headerOf(ENCODED)))
+    // A plain header isn't encoded: bsdtar lists it.
+    assert.isNull(encodedHeaderEncrypted(headerOf(PLAIN)))
+    assert.isNull(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED).subarray(0, 12)))
+  })
+
   test('finds the header at the end from the signature header', ({ assert }) => {
     const start = parseSevenZipStart(ENCODED)!
     assert.deepEqual(start, { headerStart: 157, headerSize: 33, headerCrc: start.headerCrc, archiveSize: 190 })

tests/unit/spotnet.spec.ts

@@ -146,6 +146,19 @@ test.group('spotnet / passwords in descriptions', () => {
     assert.equal(passwordFromSpot('T', '<br />Password = "abc123"<br />'), 'abc123')
   })
 
+  test('finds passwords in the phrasings found in spots that the first version missed', async ({ assert }) => {
+    const { passwordFromSpot } = await import('#lib/spotnet/index')
+    // LOG.md 2026-10-08 (10)
+    assert.equal(passwordFromSpot('T', 'Formaat: DVD5 ISO[br][br]!! LETOP Wachtwoord : oscfyajg !!'), 'oscfyajg')
+    assert.equal(passwordFromSpot('T', "Het totaal gewicht komt neer op zo'n 400GB. Deze is voorzien van een wachtwoord. Deze is: VTSm7nLdZZl5Kren[br]"), 'VTSm7nLdZZl5Kren')
+    assert.equal(passwordFromSpot('T', 'Futurama S11E10 1080P[br][br]ww in nzb: cI6f7dBV6BKkhqD[br][br]Dit was'), 'cI6f7dBV6BKkhqD')
+    assert.equal(passwordFromSpot('T', '[b]Het wachtwoord voor de nzb is: dw0022[/b][br]'), 'dw0022')
+    assert.equal(passwordFromSpot('T', 'haakinstructies.[br][br]Wachtwoord inbegrepen: wysiwyg'), 'wysiwyg')
+    assert.equal(passwordFromSpot('T', 'afgeslagen moeten worden.[br][br]wachtwoord is arn2'), 'arn2')
+    assert.equal(passwordFromSpot('T', 'Forced : No[br][br]WW:[br]tijdloos[br][br]Zag en merkte'), 'tijdloos')
+    assert.equal(passwordFromSpot('T', 'telekinetische krachten beschikt.[br]WW carrie[br][br]Multisubs'), 'carrie')
+  })
+
   test('ignores sentences about passwords', async ({ assert }) => {
     const { passwordFromSpot } = await import('#lib/spotnet/index')
     for (const description of [
@@ -158,6 +171,14 @@ test.group('spotnet / passwords in descriptions', () => {
       'Wachtwoord: geen',
       'WW: zie nzb',
       'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!',
+      'Mijn wachtwoord is hier direct te vinden, je moet het wel zelf overtypen',
+      'Bringing History to Life - Greatest Battles of WW II, Sept 2026',
+      'WW II',
+      'WW Disc 1 = 5YIMKDaoTaKe8WIWEKUF',
+      'U kunt eigenaars- en gebruikerswachtwoorden aan uw pdf toevoegen met behulp van 256-bits encryptie.',
+      'APP MISC - SHAREit v6.27.49 ww 4062749 arm64 v8a.apk',
+      'Het ww staat op het plaatje, type het EXACT zo in.',
+      'Wachtwoord:[br]Deze download heeft er geen',
     ]) {
       assert.isNull(passwordFromSpot('T', description), description)
     }