Rebuild spec: d and e decided; coordinator prefetches ranges into a spool

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.adonisjs/server/controllers.ts

@@ -9,4 +9,5 @@ export const controllers = {
   Newznab: () => import('#controllers/newznab_controller'),
   Profile: () => import('#controllers/profile_controller'),
   Releases: () => import('#controllers/releases_controller'),
+  Seed: () => import('#controllers/seed_controller'),
 }

.gitea/workflows/deploy.yaml

@@ -0,0 +1,81 @@
+# Tests every push; a push to master that passes is deployed to homelab.
+#
+# Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
+# run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
+# which fetches the commit from this repo, builds the image, runs the
+# migrations and recreates the app services (never postgres or redis).
+name: Test and deploy
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: indexer
+          POSTGRES_PASSWORD: indexer
+          POSTGRES_DB: usenet_indexer
+        options: >-
+          --health-cmd "pg_isready -U indexer -d usenet_indexer"
+          --health-interval 5s
+          --health-retries 20
+      redis:
+        image: redis:8
+    env:
+      TZ: UTC
+      PORT: 3333
+      HOST: 0.0.0.0
+      NODE_ENV: test
+      LOG_LEVEL: warn
+      APP_KEY: ci-only-app-key-not-a-secret-000000
+      APP_URL: http://localhost:3333
+      SESSION_DRIVER: memory
+      DB_CONNECTION: pg
+      DB_HOST: postgres
+      DB_PORT: 5432
+      DB_USER: indexer
+      DB_PASSWORD: indexer
+      DB_DATABASE: usenet_indexer
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      # Tests never connect to a news server.
+      NNTP_HOST: localhost
+      NNTP_PORT: 563
+      NNTP_USER: ci
+      NNTP_PASSWORD: ci
+      NNTP_SECURE: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci
+      - name: Type check
+        run: npx tsc --noEmit
+      - name: Migrate
+        run: node ace migration:run --force
+      - name: Test
+        run: node ace test
+
+  deploy:
+    needs: test
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to homelab
+        env:
+          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+          DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
+        run: |
+          install -m 700 -d ~/.ssh
+          printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
+          ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
+            daan@10.206.0.175 "deploy ${{ github.sha }}"

REBUILD.md

@@ -0,0 +1,125 @@
+# Rebuild: header-first indexer (spec, draft for approval)
+
+Branch `rebuild`. Phase 1 finds releases from headers and makes their NZBs.
+Everything learned from articles (names from yEnc headers, par2, archive
+listings, encryption, spots) is phase 2 and adds to what phase 1 stored; it
+never rewrites it. Sources: CONCEPT.md, LOG.md 2026-10-08 (13)–(14).
+
+## Target
+
+7,000 days (all of Eweka's retention, back to 2008-08) of the 1000 largest
+`alt.binaries.*` groups. Estimated: 528 B overview lines; 435 B in their
+main group; 140 B kept segments in ~1.25 B files and ~115 M releases after
+dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
+≥ 25 B overview lines/day (~50 connections), so ~3 weeks.
+
+## Parts
+
+1. **Coordinator** (server). Plans each group's history as ranges of 2 M
+   articles, leases them to workers, tracks progress and retries. The
+   group list (owner rule) and the garbage signature table go with every
+   lease. Fetches XOVER for the workers (relay: workers have no NNTP
+   account; the gzip stream is passed on as is).
+
+   **Prefetch**: the coordinator fetches the next ranges ahead of the
+   workers, in chunks of 50 k lines and in order, into a spool on disk
+   (~106 MB per 2 M range at ~53 B per line; on the HDD, written and read
+   sequentially). It keeps about two ranges ahead per active lease, capped
+   by size (default 10 GB), and fetches with all its header connections all
+   the time, independent of how fast workers are. A lease goes to a range
+   that is already (partly) spooled; a worker reads its chunks from the
+   spool, waiting only for chunks still being fetched, and falls back to a
+   direct relay fetch for a range that isn't spooled. Spooled chunks are
+   deleted when their range's results are stored, and fetched again if a
+   range is leased again later.
+   Order: oldest era first (2008–2018 holds 63% of the readable files on
+   17% of the lines), then forward.
+
+2. **Edge worker** (node, TypeScript reusing `lib/`; Rust later only if
+   CPU-bound). Per lease: compressed chunks of 10–50 k lines, in order →
+   parse subjects → skip crossposts outside their owner group (of the
+   indexed groups in the article's Xref, the one with the lowest priority
+   number; see the group list) → group articles into files → files
+   into releases (`lib/release` rules) → drop garbage (random name +
+   dropped signature or tool) → set aside per-article random subjects and
+   obfuscated subjects (counted only; phase 2). Returns per batch: release
+   chunks, edge fragments, probe candidates, counters. Holds no state beyond
+   its lease.
+
+   **Probe candidates**: per release, the articles phase 2 should read, as
+   (release key, kind, message-id, file in its chunk): the first article of
+   the first archive volume (`.part01.rar`, `.rar`, `.7z.001`), of the par2
+   index (smallest `.par2`), and of a lone random-named file; for a 7z set
+   also its last volume (the listing is in the tail). At most a few per
+   release (~115 M releases → ~150–250 M articles), not one per file. Per-
+   article and obfuscated subjects get none (they need every article's body).
+
+3. **Chunk store** (bulk, append-only). One record per release per batch:
+   its files' names, sizes and segment lists, compactly encoded (template
+   codec for message-ids, zstd). Written once, never updated; a re-run of a
+   range replaces that range's chunks (no per-file dedup). ~3 TB.
+
+4. **Release catalog** (hot, Postgres on NVMe). One row per release: key,
+   name, poster, groups, posted_at, file count, size, segments, expected
+   file count (`[i/N]`), has-par2, readable-name flag, state. Updated with
+   aggregates once per batch (one upsert per release per batch), plus a
+   small name index for search. ~115 M rows, ~50–100 GB.
+
+5. **Rules and recovery**. What a worker skips isn't stored (message-ids
+   are the bulk: garbage alone would be ~2.5 TB); a skipped range is read
+   again instead (headers are cheap: ~53 B per overview line compressed).
+   - *Per range, per rule and signature*: how many articles and files the
+     rule dropped (garbage per (tool, group), per-article random subjects,
+     obfuscated subjects, and crossposts per owner group). A few rows per
+     range.
+   - *Rule versions*: every range records the version of the rule set it ran
+     with. Retracting or changing a rule makes the ranges where it matched
+     stale; the coordinator leases them again for that rule only, and the
+     worker returns just what the rule had dropped.
+   - *Sample*: ~1% of what each rule drops (by a hash of the release key) is
+     kept with full file details and probe candidates, so a rule can be
+     checked, and its losses measured in phase 2, before it is retracted.
+
+6. **Probe queue**. The candidates, in the catalog's database: one small row
+   each (release, kind, message-id, chunk reference), priority by the
+   release's state (settled first) and kind (random names before readable
+   ones). Phase 2 takes them from here; phase 1 only fills it.
+
+7. **Fragments**. Files crossing a range edge: kept by range and article
+   number; merged when the neighbouring range is done, then written as a
+   chunk. Never staged per segment.
+
+8. **Settling**. A release is settled when the ranges covering its posting
+   span are done and no chunk arrived for a day: completeness is computed
+   then (expected vs found files and segments). Phase 2 picks its work from
+   settled releases.
+
+9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
+   search (names) and get. Web UI later.
+
+## Group list
+
+Editable at any time: an added group gets its ranges planned, a removed one
+no new leases. Each group gets a priority number when it is added (the first
+list by size, later additions after it); a crosspost belongs to the indexed
+group in its Xref with the lowest number. So adding a group never takes
+articles from groups already done, and removing one re-reads only the ranges
+of other groups whose counts show crossposts skipped for it (part 5). Start:
+the 1000 largest `alt.binaries.*` groups by LIST ACTIVE (~266 k ranges of
+2 M articles; boneless alone 80 k).
+
+## Not in phase 1
+
+Inspection and archive walks, par2/SFV naming, spots, NZB imports, the
+after-probe garbage deletes, live indexing (the current indexer keeps
+running and does those meanwhile).
+
+## Decisions
+
+a. Chunk payload: a write-once `bytea` table in Postgres, partitioned by
+   month, on the HDD; files on disk only if writes still limit. **Accepted.**
+b. A separate database on the same server. **Accepted.**
+c. Group list: editable, owner by priority number (above). **Accepted.**
+d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
+e. Headers through the coordinator's relay, with prefetch (part 1).
+   **Accepted.**

app/controllers/newznab_controller.ts

@@ -101,7 +101,9 @@ export default class NewznabController {
         postedAt: r.postedAt ? new Date(r.postedAt) : null,
         poster: r.poster,
         groups: r.groups ?? [],
-        password: Boolean(r.encrypted),
+        // 1 only when a password is needed that we can't give; a known one is in our NZB.
+        password: r.passwordState === 'missing',
+        description: r.spotDescription ?? undefined,
         season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
         episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
       }

app/controllers/releases_controller.ts

@@ -3,8 +3,14 @@ import SearchService, { type SortOrder } from '#services/SearchService'
 
 const flag = (value: unknown) => value === '1' || value === 'true' || value === true
 
+/** A size in MB from the query string, as bytes. */
+function megabytes(value: unknown): number | undefined {
+  const mb = Number(value)
+  return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined
+}
+
 export default class ReleasesController {
-  /** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&source=&poster=&sort=&page=&per_page= */
+  /** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&min_size=&max_size=(MB)&ext=mkv,flac&source=&poster=&sort=&page=&per_page= */
   async index({ request }: HttpContext) {
     const qs = request.qs()
     return SearchService.search({
@@ -15,9 +21,14 @@ export default class ReleasesController {
       includePar2Only: flag(qs.par2_only),
       source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
       poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
+      minSize: megabytes(qs.min_size),
+      maxSize: megabytes(qs.max_size),
+      extensions: typeof qs.ext === 'string' && qs.ext ? qs.ext.split(/[\s,]+/) : undefined,
       sort: qs.sort as SortOrder | undefined,
       page: Number(qs.page) || 1,
       perPage: Number(qs.per_page) || 50,
+      // The page shows "1,000+"; counting further made filter clicks take seconds.
+      countLimit: 1000,
     })
   }
 
@@ -37,6 +48,16 @@ export default class ReleasesController {
     return response.send(nzb.xml)
   }
 
+  /** GET /api/v1/releases/:id/image: the image of the release's spot, read from Usenet. */
+  async image({ params, response }: HttpContext) {
+    const image = await SearchService.spotImage(Number(params.id))
+    if (!image) return response.notFound({ error: 'No image' })
+    response.header('Content-Type', image.type)
+    // A spot's image never changes.
+    response.header('Cache-Control', 'public, max-age=604800, immutable')
+    return response.send(image.data)
+  }
+
   /** GET /api/v1/stats */
   async stats() {
     return SearchService.stats()

app/controllers/seed_controller.ts

@@ -0,0 +1,156 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator'
+import SeedFetchPool from '#services/SeedFetchPool'
+import RemoteContentService, { ArticleRelayOff, JobLost } from '#services/RemoteContentService'
+import type { SeedBatch } from '#lib/seed/wire'
+
+/**
+ * The seed API (CONCEPT.md D38). Nodes make outgoing HTTPS requests only:
+ * lease a range, post batches of what they found, then finish or release
+ * the range. Bodies may be gzip-compressed (Content-Encoding: gzip).
+ */
+export default class SeedController {
+  /** POST /api/v1/seed/lease → 200 { range } or 204 when there's nothing to do. */
+  async lease({ seedNode, response }: HttpContext) {
+    const range = await SeedOrchestrator.lease(seedNode.id)
+    return range ? { range } : response.noContent()
+  }
+
+  /** POST /api/v1/seed/ranges/:id/batches with a SeedBatch. */
+  async batch({ seedNode, params, request, response }: HttpContext) {
+    const batch = request.body() as SeedBatch
+    if (!Number.isSafeInteger(batch?.next) || !Array.isArray(batch.files) || !Array.isArray(batch.fragments)) {
+      return response.badRequest({ error: 'Expected { next, headers, files[], fragments[] }' })
+    }
+    return this.leased(response, async () => {
+      const stored = await SeedOrchestrator.batch(seedNode.id, Number(params.id), {
+        ...batch,
+        headers: Number(batch.headers) || 0,
+        obfuscated: Number(batch.obfuscated) || 0,
+        deferred: Number(batch.deferred) || 0,
+      })
+      return { stored }
+    })
+  }
+
+  /**
+   * GET /api/v1/seed/ranges/:id/chunk?from=&to= → the overviews of those
+   * articles as a zlib/gzip stream (relay nodes fetch through the indexer).
+   */
+  async chunk({ seedNode, params, request, response }: HttpContext) {
+    if (SeedFetchPool.size === 0) return response.serviceUnavailable({ error: 'Relay is off (SEED_NNTP_CONNECTIONS=0); fetch directly' })
+    const from = Number(request.input('from'))
+    const to = Number(request.input('to'))
+    if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to)) return response.badRequest({ error: 'Expected ?from=&to=' })
+    return this.leased(response, async () => {
+      try {
+        const raw = await SeedOrchestrator.chunk(seedNode.id, Number(params.id), from, to)
+        response.header('content-type', 'application/octet-stream')
+        return response.send(raw)
+      } catch (error) {
+        if (error instanceof RangeError) return response.badRequest({ error: error.message })
+        throw error
+      }
+    })
+  }
+
+  /** POST /api/v1/seed/ranges/:id/done */
+  async done({ seedNode, params, response }: HttpContext) {
+    return this.leased(response, async () => {
+      await SeedOrchestrator.finish(seedNode.id, Number(params.id))
+      return { ok: true }
+    })
+  }
+
+  /** POST /api/v1/seed/ranges/:id/release { error?, uncounted? } */
+  async release({ seedNode, params, request, response }: HttpContext) {
+    const error = request.input('error')
+    const uncounted = request.input('uncounted') === true
+    return this.leased(response, async () => {
+      await SeedOrchestrator.release(seedNode.id, Number(params.id), typeof error === 'string' ? error : undefined, uncounted)
+      return { ok: true }
+    })
+  }
+
+  /**
+   * POST /api/v1/seed/inspect/lease { max } → 200 { jobs } or 204: content
+   * inspection and archive walks for the node (CONCEPT.md D41).
+   */
+  async inspectLease({ seedNode, request, response }: HttpContext) {
+    const max = Math.min(10, Math.max(1, Number(request.input('max')) || 1))
+    const jobs = await RemoteContentService.lease(seedNode.id, max)
+    return jobs.length > 0 ? { jobs } : response.noContent()
+  }
+
+  /**
+   * POST /api/v1/seed/inspect/done { queue, jobId, token, result } where
+   * result is the probe as a JSON string (the body parser would turn the
+   * empty strings inside it into nulls).
+   */
+  async inspectDone({ request, response }: HttpContext) {
+    const { queue, jobId, token, result } = request.only(['queue', 'jobId', 'token', 'result'])
+    if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string' || typeof result !== 'string') {
+      return response.badRequest({ error: 'Expected { queue, jobId, token, result: JSON string }' })
+    }
+    return this.job(response, () => RemoteContentService.finish(queue, jobId, token, JSON.parse(result)))
+  }
+
+  /** POST /api/v1/seed/inspect/failed { queue, jobId, token, error } */
+  async inspectFailed({ request, response }: HttpContext) {
+    const { queue, jobId, token, error } = request.only(['queue', 'jobId', 'token', 'error'])
+    if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
+      return response.badRequest({ error: 'Expected { queue, jobId, token, error }' })
+    }
+    return this.job(response, () => RemoteContentService.fail(queue, jobId, token, String(error ?? 'unknown error')))
+  }
+
+  /** POST /api/v1/seed/inspect/release { queue, jobId, token }: a stopping node gives a job back. */
+  async inspectRelease({ request, response }: HttpContext) {
+    const { queue, jobId, token } = request.only(['queue', 'jobId', 'token'])
+    if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
+      return response.badRequest({ error: 'Expected { queue, jobId, token }' })
+    }
+    return this.job(response, () => RemoteContentService.release(queue, jobId, token))
+  }
+
+  /** GET /api/v1/seed/articles?id=<message id> → the raw article body; 410 when the server doesn't have it. */
+  async article({ request, response }: HttpContext) {
+    const id = request.input('id')
+    if (typeof id !== 'string' || id.length === 0 || id.length > 250) return response.badRequest({ error: 'Expected ?id=<message id>' })
+    try {
+      const body = await RemoteContentService.article(id)
+      if (!body) return response.gone({ error: 'No such article' })
+      response.header('content-type', 'application/octet-stream')
+      return response.send(body)
+    } catch (error: any) {
+      if (error instanceof ArticleRelayOff) return response.serviceUnavailable({ error: error.message })
+      return response.badGateway({ error: `NNTP: ${error.message}` })
+    }
+  }
+
+  /** GET /api/v1/seed/status */
+  async status() {
+    return SeedOrchestrator.status()
+  }
+
+  /** 409 when the job's lock is gone (it was given to another node meanwhile). */
+  private async job(response: HttpContext['response'], action: () => Promise<unknown>) {
+    try {
+      await action()
+      return { ok: true }
+    } catch (error) {
+      if (error instanceof JobLost) return response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
+  /** 409 when the range was given to another node meanwhile; the node should drop it. */
+  private async leased(response: HttpContext['response'], action: () => Promise<unknown>) {
+    try {
+      return await action()
+    } catch (error) {
+      if (error instanceof LeaseLost) return response.conflict({ error: error.message })
+      throw error
+    }
+  }
+}

app/middleware/seed_node_middleware.ts

@@ -0,0 +1,26 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import type { NextFn } from '@adonisjs/core/types/http'
+import SeedOrchestrator from '#services/SeedOrchestrator'
+
+declare module '@adonisjs/core/http' {
+  interface HttpContext {
+    /** The seeding node that made this request (CONCEPT.md D38). */
+    seedNode: { id: number; name: string }
+  }
+}
+
+/**
+ * Seed API requests carry a node's bearer token (`node ace seed:token
+ * <name>`); anything else gets 401. The seed API is the only part of the app
+ * reachable from the internet (traefik routes only /api/v1/seed there).
+ */
+export default class SeedNodeMiddleware {
+  async handle(ctx: HttpContext, next: NextFn) {
+    const header = ctx.request.header('authorization') ?? ''
+    const token = header.startsWith('Bearer ') ? header.slice(7).trim() : ''
+    const node = token ? await SeedOrchestrator.nodeForToken(token) : null
+    if (!node) return ctx.response.unauthorized({ error: 'Unknown or disabled seed node token' })
+    ctx.seedNode = node
+    return next()
+  }
+}

app/models/file.ts

@@ -1,6 +1,8 @@
 import { DateTime } from 'luxon'
 import { column } from '@adonisjs/lucid/orm'
 import BaseModel from '#models/base_model'
+import { loadedSegments, storedSegments } from '#services/FileStore'
+import type { Segment } from '#lib/nzb/index'
 
 export default class File extends BaseModel {
   @column({ isPrimary: true })
@@ -21,8 +23,12 @@ export default class File extends BaseModel {
   @column()
   declare parts: number
 
-  @column({ columnName: 'message_ids' })
-  declare messageIds: Record<string, any>
+  /** Stored compressed (see FileStore); the model sees the list. */
+  @column({ prepare: (value: Segment[]) => storedSegments(value), consume: (value: Buffer | null) => loadedSegments(value) })
+  declare segments: Segment[]
+
+  @column({ columnName: 'first_message_id' })
+  declare firstMessageId: string | null
 
   @column()
   declare groups: string[]

app/services/ArchiveService.ts

@@ -1,13 +1,9 @@
-import { execFile } from 'node:child_process'
-import { mkdtemp, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import path from 'node:path'
-import { promisify } from 'node:util'
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
+import FileStore from '#services/FileStore'
 import IngestService from '#services/IngestService'
-import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
-import { fileStem, rarReleaseIdentity } from '#lib/release/index'
+import type { RarInfo } from '#lib/rar/index'
+import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
 
 /** Stored per file and release; a listing of thousands of files is not useful. */
 const MAX_ENTRIES = 500
@@ -27,8 +23,9 @@ export interface ArchiveResult {
  */
 class ArchiveService {
   async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
-    const stored = { ...info, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
-    await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) })
+    const { next: _next, ...listing } = info
+    const stored = { ...listing, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
+    await FileStore.setDetails(Number(file.id), { archive: stored })
 
     let releaseId = file.release_id ?? binary?.release_id ?? null
     let grouped = false
@@ -59,7 +56,9 @@ class ArchiveService {
       update.encrypted = true
     }
     const better = releaseNameFromContents(info)
-    const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
+    // A spot's title is kept (CONCEPT.md D29).
+    const renamed =
+      release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
     if (renamed) update.name = renamed
     if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
 
@@ -68,43 +67,6 @@ class ArchiveService {
   }
 }
 
-const run = promisify(execFile)
-
-/** Tools that print one archive entry to stdout, tried in order. */
-const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
-  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
-  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
-]
-
-/**
- * The unpacked data of one entry of a complete (single-volume) archive:
- * copied out when stored, else unpacked with bsdtar (libarchive, in the
- * Docker image) or unrar. Null when that is not possible.
- */
-export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
-  const stored = storedEntryData(archive, entry)
-  if (stored) return stored
-  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
-  if (entry.size !== null && entry.size > maxBytes) return null
-
-  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
-  try {
-    const file = path.join(dir, 'archive.rar')
-    await writeFile(file, archive)
-    for (const [tool, args] of EXTRACTORS) {
-      try {
-        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
-        if (stdout.length > 0) return stdout
-      } catch (error: any) {
-        if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
-      }
-    }
-    return null
-  } finally {
-    await rm(dir, { recursive: true, force: true })
-  }
-}
-
 /**
  * A release name from an archive listing: the top-level directory shared by
  * all entries when it reads like a release name, otherwise the stem of the
@@ -112,9 +74,11 @@ export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes =
  * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
  */
 export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
-  const entries = info.entries.filter((e) => !e.directory)
-  const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0]))
-  if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) {
+  // Hidden entries name nothing: one poster adds the same 22 GB ".1917-….mkv" decoy to every upload (LOG.md 2026-10-06 (2)).
+  const listed = info.entries.filter((e) => !isHidden(e.name))
+  const entries = listed.filter((e) => !e.directory)
+  const tops = new Set(listed.map((e) => e.name.split(/[\\/]/)[0]))
+  if (tops.size === 1 && listed.some((e) => /[\\/]/.test(e.name))) {
     const [top] = tops
     if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
   }
@@ -126,6 +90,36 @@ export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string
   return candidates[0]?.stem
 }
 
+/** An entry in a hidden directory or a hidden file itself (`.x/…`, `dir/.x`). */
+export function isHidden(name: string): boolean {
+  return name.split(/[\\/]/).some((part) => part.startsWith('.'))
+}
+
+/**
+ * Releases named after a hidden archive entry before those were skipped:
+ * renamed from their stored listing (no downloads). Used by
+ * `release:rename --hidden`.
+ */
+export async function renameHiddenNamed(): Promise<number> {
+  const releases = await db
+    .from('releases')
+    .where('name', 'like', '.%')
+    .whereNot('source', 'spot')
+    .whereNotNull('contents')
+    .select('id', 'name', 'contents')
+  let renamed = 0
+  for (const release of releases) {
+    const entries: any[] = Array.isArray(release.contents) ? release.contents : []
+    const fromHidden = entries.some((e) => isHidden(e.name) && fileStem(String(e.name).split(/[\\/]/).pop() ?? '') === release.name)
+    if (!fromHidden) continue
+    const better = releaseNameFromContents({ entries })
+    if (!better || better === release.name) continue
+    await db.from('releases').where('id', release.id).update({ name: better })
+    renamed++
+  }
+  return renamed
+}
+
 /**
  * Renames random-named releases from their stored archive listing (no
  * downloads). Used by `release:rename`.
@@ -138,7 +132,7 @@ export async function renameFromStoredContents(): Promise<number> {
     .select('id', 'name', 'source', 'contents')
   let renamed = 0
   for (const release of releases) {
-    if (!looksRandom(release.name) && release.source !== 'post') continue
+    if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
     const better = releaseNameFromContents({ entries: release.contents })
     if (!better || better === release.name) continue
     await db.from('releases').where('id', release.id).update({ name: better })
@@ -147,11 +141,27 @@ export async function renameFromStoredContents(): Promise<number> {
   return renamed
 }
 
-/** Obfuscated names: one long token of mixed letters and digits, no separators. */
+/**
+ * Obfuscated names: one long token of mixed letters and digits, no
+ * separators, and the other generated shapes seen on Usenet (LOG.md
+ * 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
+ */
 export function looksRandom(name: string): boolean {
   // An unknown short extension (e.g. ".bin") says nothing about the name.
   name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
+  // VIDEO_TS, Sample, CD1: real words, but no name.
+  if (isGenericName(name)) return true
   if (/^[0-9a-f]{16,}$/i.test(name)) return true
+  // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
+  if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
+  // UUIDs
+  if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
+  // 1650777167 (yEncBin), "83111763 n" (rclone uploads)
+  if (/^\d{6,}( n)?$/.test(name)) return true
+  // prvjr7hapd0i1ny14pz4, and rclone's long base32 names
+  if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
+  // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
+  if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
   if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
   // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
   return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6

app/services/ArchiveWalkService.ts

@@ -0,0 +1,160 @@
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import { segmentsOf } from '#services/FileStore'
+import { looksRandom, releaseNameFromContents } from '#services/ArchiveService'
+import { rarVolume } from '#lib/content/index'
+import type { RarEntry, RarInfo } from '#lib/rar/index'
+import { probeWalk, type WalkProbe, type WalkTask } from '#lib/content/probe'
+
+/** Stored per release, as for a first-volume listing (ArchiveService). */
+const MAX_ENTRIES = 500
+
+/** Listed sizes below this share of the release's size: the first segment listed only part of it. */
+const COVERED = 0.7
+
+/** Inspection queues a walk this long after the first volume, when the release has had time to complete. */
+const WALK_DELAY_MS = 30 * 60 * 1000
+
+export interface WalkOutcome {
+  releaseId: number
+  /** Nothing downloaded: why. */
+  skipped?: 'missing' | 'covered' | 'volumes'
+  entries?: number
+  before?: number
+  complete?: boolean
+  stopped?: string
+  segments?: number
+  renamed?: string
+}
+
+/**
+ * Complete archive listings by walking RAR headers (CONCEPT.md D40). The
+ * first segment of a first volume lists the entries that start in it: for
+ * a set of many small files (an audiobook's MP3s, a music album) that is
+ * the first one or two. The walk reads one segment per further entry and
+ * volume, at low priority: its own queue, one job at a time per collection
+ * worker, and NNTP connections only when inspection isn't waiting for one.
+ */
+class ArchiveWalkService {
+  /** Segments one walk may download (~0.7 MB each). */
+  readonly maxSegments = env.get('ARCHIVE_WALK_MAX_SEGMENTS', 400)
+
+  /** A listing that leaves most of the release unaccounted for. */
+  worthWalking(entries: number, listedBytes: number, releaseSize: number): boolean {
+    if (entries === 0 || entries >= MAX_ENTRIES) return false
+    return listedBytes < COVERED * releaseSize
+  }
+
+  /** After inspection listed a first RAR volume whose listing ends inside the set. */
+  async queueAfterInspection(releaseId: number | null, info: RarInfo) {
+    if (!releaseId || info.format === '7z' || info.firstVolume === false) return
+    if (!info.truncated || info.headersEncrypted || info.entries.length === 0) return
+    await QueueService.archiveWalkQueue.add('walk', { releaseId }, { jobId: `walk-${releaseId}`, delay: WALK_DELAY_MS })
+  }
+
+  /**
+   * The database half before a walk (D41): the release's volumes, or the
+   * outcome when there's nothing to walk.
+   */
+  async prepare(releaseId: number): Promise<{ task: WalkTask } | { outcome: WalkOutcome }> {
+    const release = await db.from('releases').where('id', releaseId).select('id', 'size', 'contents').first()
+    if (!release) return { outcome: { releaseId, skipped: 'missing' } }
+    const before = Array.isArray(release.contents) ? release.contents : []
+    const listed = before.reduce((sum: number, e: any) => sum + Number(e.size ?? 0), 0)
+    if (!this.worthWalking(before.length, listed, Number(release.size))) {
+      return { outcome: await this.done(releaseId, { releaseId, skipped: 'covered' }) }
+    }
+    const volumes = await this.volumes(releaseId)
+    if (!volumes) return { outcome: await this.done(releaseId, { releaseId, skipped: 'volumes' }) }
+    return {
+      task: {
+        kind: 'walk',
+        releaseId,
+        volumes: volumes.map((file) => segmentsOf(file).map(({ number, id }) => ({ number, id }))),
+        maxSegments: this.maxSegments,
+      },
+    }
+  }
+
+  /** The database half after a walk: the fuller listing, the password flag and a better name. */
+  async apply(releaseId: number, walked: WalkProbe): Promise<WalkOutcome> {
+    const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'contents', 'encrypted').first()
+    if (!release) return { releaseId, skipped: 'missing' }
+    const before = Array.isArray(release.contents) ? release.contents : []
+    const outcome: WalkOutcome = {
+      releaseId,
+      before: before.length,
+      entries: walked.entries.length,
+      complete: walked.complete,
+      stopped: walked.stopped,
+      segments: walked.segments,
+    }
+    if (walked.entries.length > before.length) {
+      const contents = walked.entries.slice(0, MAX_ENTRIES)
+      const update: Record<string, unknown> = {
+        contents: JSON.stringify(contents),
+        encrypted: Boolean(release.encrypted) || walked.headersEncrypted || walked.entries.some((e) => e.encrypted),
+      }
+      const better = releaseNameFromContents({ entries: walked.entries as RarEntry[] })
+      // The rule of ArchiveService.record: random and post-subject names give way, a spot's title stays.
+      if (better && better !== release.name && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post')) {
+        update.name = better
+        outcome.renamed = better
+      }
+      await db.from('releases').where('id', releaseId).update(update)
+    }
+    return this.done(releaseId, outcome)
+  }
+
+  /** Walks a release here, on the inspection pool's connections when inspection isn't waiting for one. */
+  async walk(releaseId: number): Promise<WalkOutcome> {
+    const prepared = await this.prepare(releaseId)
+    if ('outcome' in prepared) return prepared.outcome
+    return this.apply(releaseId, await probeWalk(prepared.task, fetchLow))
+  }
+
+  private async done(releaseId: number, outcome: WalkOutcome) {
+    await db.from('releases').where('id', releaseId).update({ contents_walked_at: new Date() })
+    return outcome
+  }
+
+  /**
+   * The release's RAR volumes in order, from volume 1 up to the first gap.
+   * Null without a first volume (e.g. volumes with random names).
+   */
+  private async volumes(releaseId: number): Promise<any[] | null> {
+    const files = await db.from('files').where('release_id', releaseId).select('id', 'filename', 'real_name', 'segments')
+    const sets = new Map<string, Map<number, any>>()
+    for (const file of files) {
+      const volume = rarVolume(String(file.real_name ?? file.filename))
+      if (!volume || segmentsOf(file).length === 0) continue
+      const set = sets.get(volume.base) ?? new Map()
+      if (!set.has(volume.number)) set.set(volume.number, file)
+      sets.set(volume.base, set)
+    }
+    // A release holds one set, now and then a second (a sample, subs): the biggest.
+    const set = [...sets.values()].sort((a, b) => b.size - a.size)[0]
+    if (!set?.has(1)) return null
+    const ordered: any[] = []
+    for (let n = 1; set.has(n); n++) ordered.push(set.get(n))
+    return ordered
+  }
+}
+
+/** One article body on a pooled connection, behind inspection's waiting jobs. */
+async function fetchLow(messageId: string): Promise<Buffer> {
+  const conn = await NntpService.acquire(true)
+  let failed = false
+  try {
+    return await NntpService.timed(NntpService.fetchBody(conn, messageId), 'BODY')
+  } catch (error) {
+    failed = true
+    throw error
+  } finally {
+    NntpService.release(conn, failed)
+  }
+}
+
+export default new ArchiveWalkService()

app/services/BackfillService.ts

@@ -0,0 +1,112 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import NntpService from '#services/NntpService'
+import FetchRangeService from '#services/FetchRangeService'
+import { articleAtDate } from '#lib/nntp/date_search'
+
+/** Articles XOVERed to find a posting date near an article number. */
+const PROBE = 200n
+
+/**
+ * Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
+ * where live indexing began back to the article posted N days ago; `tick`
+ * (every scheduler run) queues low-priority, header-only fetch ranges from
+ * that cursor downwards, newest first, while few are waiting and the
+ * database stays within its size limits.
+ */
+class BackfillService {
+  /** Sets the group's backfill to reach back `days` days. Returns its range. */
+  async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
+    const group = await db.from('groups').where('name', groupName).first()
+    if (!group) throw new Error(`Unknown group ${groupName}`)
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
+      const dateOf = async (n: bigint) => {
+        const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
+        const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
+        const date = header ? Date.parse(header.date) : Number.NaN
+        return Number.isNaN(date) ? null : date
+      }
+      const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
+      // Live indexing began a little before the group was added: continue from there.
+      const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
+      const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
+      await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
+      return next > until ? { from: until, to: next } : null
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+
+  /** Stops a group's backfill (queued ranges still run). */
+  async stop(groupName: string) {
+    await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
+  }
+
+  /** Queues the next backfill ranges. Returns how many. */
+  async tick(): Promise<number> {
+    const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
+    const groups = await db
+      .from('groups')
+      .where('active', true)
+      .whereNotNull('backfill_until_id')
+      .whereRaw('backfill_next_id > backfill_until_id')
+      .orderBy('id')
+    if (groups.length === 0) return 0
+
+    const room = await this.room(maxDefaultGb, maxHddGb)
+    if (room !== true) {
+      logger.warn(`Backfill paused: ${room}.`)
+      return 0
+    }
+    const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
+    let free = maxQueued - Number(count)
+    let queued = 0
+    // Round-robin over the groups, one range each per pass.
+    while (free > 0 && groups.length > 0) {
+      for (const group of [...groups]) {
+        if (free <= 0) break
+        const next = BigInt(group.backfill_next_id)
+        const until = BigInt(group.backfill_until_id)
+        const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
+        await FetchRangeService.queue(group, start, next, { backfill: true })
+        group.backfill_next_id = (start - 1n).toString()
+        await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
+        free--
+        queued++
+        if (start - 1n <= until) {
+          groups.splice(groups.indexOf(group), 1)
+          logger.info(`${group.name}: backfill fully queued.`)
+        }
+      }
+    }
+    return queued
+  }
+
+  /** True when both parts of the database are under their limits, else why not. */
+  /**
+   * Whether the database has room for more history: the default tablespace
+   * (NVMe) and tablespace `hdd` against BACKFILL_MAX_NVME_GB/_HDD_GB. Seed
+   * leases (CONCEPT.md D38) check the same limits.
+   */
+  async room(
+    maxDefaultGb = indexerConfig.backfill.maxDefaultGb,
+    maxHddGb = indexerConfig.backfill.maxHddGb
+  ): Promise<true | string> {
+    const { rows } = await db.rawQuery(
+      `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
+    )
+    const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
+    if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
+    if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
+    return true
+  }
+}
+
+export default new BackfillService()

app/services/BinaryFileService.ts

@@ -0,0 +1,144 @@
+import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
+import ContentService from '#services/ContentService'
+import InspectionQueue from '#services/InspectionQueue'
+
+export interface BinaryFilesResult {
+  /** New files. */
+  saved: number
+  /** Binaries whose file already existed (same first segment or file_key). */
+  existing: number
+  /** Conflicted, gone or already linked: nothing to do. */
+  skipped: number
+  /** Not all parts stored yet (the job is retried for these). */
+  incomplete: number[]
+  inspections: number
+}
+
+/**
+ * Turns completed binaries into files (worker:file), a batch at a time
+ * (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and
+ * files that exist already, one INSERT for the new files (so the files
+ * statement trigger updates each release once), and one UPDATE and DELETE
+ * for the binaries' links and copied parts. It was one job of ~8 queries per
+ * binary, ~700 a minute.
+ */
+class BinaryFileService {
+  async store(binaryIds: number[]): Promise<BinaryFilesResult> {
+    const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 }
+    const ids = [...new Set(binaryIds)]
+    const binaries = await db
+      .from('binaries')
+      .whereIn('id', ids)
+      .whereNull('conflicted_at')
+      .whereNull('file_id')
+      .select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts')
+    result.skipped = ids.length - binaries.length
+    if (binaries.length === 0) return result
+
+    const partsOf = new Map<number, { number: number; id: string; size: number }[]>()
+    const parts = await db
+      .from('parts')
+      .whereIn('binary_id', binaries.map((b) => b.id))
+      .orderBy([{ column: 'binary_id' }, { column: 'number' }])
+      .select('binary_id', 'number', 'message_id', 'bytes')
+    for (const part of parts) {
+      const list = partsOf.get(Number(part.binary_id)) ?? []
+      list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) })
+      partsOf.set(Number(part.binary_id), list)
+    }
+    const complete = binaries.filter((b) => {
+      const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts
+      if (!ok) result.incomplete.push(Number(b.id))
+      return ok
+    })
+    if (complete.length === 0) return result
+
+    // Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list).
+    const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id)
+    const byFirst = new Map<string, number>()
+    for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) {
+      byFirst.set(row.first_message_id, Number(row.id))
+    }
+    const fileIdOf = new Map<number, number>()
+    const fresh = complete.filter((b) => {
+      const existing = byFirst.get(partsOf.get(Number(b.id))![0].id)
+      if (existing) fileIdOf.set(Number(b.id), existing)
+      return !existing
+    })
+
+    const now = new Date()
+    // file_key is unique: a second completion of the same upload inserts nothing.
+    const inserted = await FileStore.insert(
+      fresh.map((b) => {
+        const segments = partsOf.get(Number(b.id))!
+        return {
+          file_key: b.file_key,
+          filename: b.name,
+          poster: b.poster,
+          date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(),
+          parts: segments.length,
+          segments,
+          groups: JSON.stringify(b.groups),
+          release_id: b.release_id,
+          real_name: b.real_name,
+          size: segments.reduce((sum, s) => sum + s.size, 0),
+          created_at: now,
+          updated_at: now,
+        }
+      })
+    )
+    const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id]))
+    const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key)
+    const byKey = new Map<string, number>()
+    if (missing.length > 0) {
+      for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id))
+    }
+    for (const b of fresh) {
+      const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
+      if (id) fileIdOf.set(Number(b.id), id)
+    }
+    const fresher = new Map(fresh.map((b) => [b.file_key, b]))
+    result.saved = inserted.length
+    result.existing = fileIdOf.size - inserted.length
+
+    // Links, locked in file_key order like ingest's upsert, so the two can't deadlock.
+    const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key))
+    if (linked.length > 0) {
+      await db.transaction(async (trx) => {
+        await trx.rawQuery(
+          `SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
+          [linked.map((b) => b.id)]
+        )
+        // Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16).
+        const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id))
+        await trx.rawQuery(
+          `UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END
+           FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`,
+          [linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))]
+        )
+      })
+      // Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
+      await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)])
+    }
+
+    // As for one file before: an SFV posted earlier may list them (D34), then inspection.
+    const added = inserted.flatMap((row) => {
+      const b = row.file_key ? fresher.get(row.file_key) : undefined
+      return b
+        ? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }]
+        : []
+    })
+    if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file))
+    // By the posted name, as before: inspection skips later volumes and readable media names anyway.
+    const jobs = added
+      .filter((file) => ContentService.isCandidate(file.posted))
+      .map((file) => file.id)
+    if (jobs.length > 0) await InspectionQueue.add(jobs)
+    result.inspections = jobs.length
+    return result
+  }
+}
+
+export default new BinaryFileService()

app/services/ContentService.ts

@@ -1,17 +1,31 @@
-import { createHash } from 'node:crypto'
 import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
 import NntpService from '#services/NntpService'
 import Par2Service, { type Par2Result } from '#services/Par2Service'
-import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
+import ArchiveService, { looksRandom, type ArchiveResult } from '#services/ArchiveService'
 import NameService from '#services/NameService'
-import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
+import { decodeSegment } from '#lib/yenc/index'
 import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
-import { sniffContentType, type ContentType } from '#lib/content/index'
-import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
-import { fileStem } from '#lib/release/index'
+import {
+  isArchiveStart,
+  isLaterVolume,
+  NICOVIDEO_FILE,
+  NICOVIDEO_RELEASE,
+  nicovideoReleaseName,
+  type ContentType,
+  type NicovideoInfo,
+} from '#lib/content/index'
+import { chunkMd5, probeInspection, type InspectProbe, type InspectTask, type TaskSegment } from '#lib/content/probe'
+import type { Segment } from '#lib/nzb/index'
+import { fileStem, releaseIdentity } from '#lib/release/index'
+import IngestService from '#services/IngestService'
 import { looksLikeFilename } from '#lib/subject/index'
-
-const FIRST_CHUNK = 16 * 1024
+import { sevenZipVolume } from '#lib/sevenzip/index'
+import SevenZipService from '#services/SevenZipService'
+import SfvService from '#services/SfvService'
+import ArchiveWalkService from '#services/ArchiveWalkService'
+import FileStore, { segmentsOf } from '#services/FileStore'
+import GarbageFilter from '#services/GarbageFilter'
 
 /**
  * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
@@ -22,11 +36,6 @@ export function firstChunkMd5(body: Buffer, fileSize: number): string | undefine
   return chunkMd5(decodeSegment(body).data, fileSize)
 }
 
-function chunkMd5(data: Buffer, fileSize: number): string | undefined {
-  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
-  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
-}
-
 export interface ContentResult {
   par2?: Par2Result
   archive?: ArchiveResult
@@ -35,6 +44,14 @@ export interface ContentResult {
   nzb?: NzbImportResult
   /** NZBs read from inside an archive. */
   archiveNzbs?: NzbImportResult[]
+  /** Files an SFV's listing moved into its release. */
+  sfvMoved?: number
+  /** A niconico capture's release name from its API response (D34). */
+  nicovideo?: string
+  /** The real name from the yEnc header, for a file posted under a random subject (D36). */
+  yencName?: string
+  /** Why the file's release was deleted as garbage after this probe (D42). */
+  garbage?: string
 }
 
 /**
@@ -43,92 +60,137 @@ export interface ContentResult {
  * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
  * - par2 files are recognised by content and name every file of their set,
  * - RAR headers give the release grouping, name and contents.
+ * Inspection is prepare (database) → probe (downloads and parsing,
+ * lib/content/probe; here or on a seeding node, D41) → apply (database).
  */
 class ContentService {
   /** Random or missing names, and named first RAR volumes (for contents and password flag). */
   isCandidate(name: string) {
+    // Even under a random stem: the first volume and the index .par2 are inspected instead.
+    if (isLaterVolume(name)) return false
     if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
+    if (isArchiveStart(name)) return true
     if (/\.nzb$/i.test(name)) return true
+    // SFVs name the files of their release; niconico captures carry their title (D34).
+    if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
+    // Every 7z volume: the set is listed when its last volume is in (see inspect).
+    if (sevenZipVolume(name) !== null) return true
     return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
   }
 
-  /** Downloads a whole (small) file whose first segment is already decoded. */
-  private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
-    const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
-    const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
-    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
-    if (parts.length === 1) return first
+  /**
+   * Uploads that randomise the subject per article often keep the real file
+   * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
+   * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
+   * leaves its weak one-file release for the release its name gives, where
+   * the other files of the upload gather. Returns the name, or undefined.
+   */
+  private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
+    const current = String(file.real_name ?? file.filename)
+    if (file.real_name || !looksRandom(fileStem(current))) return undefined
+    if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined
 
-    let assembled: YencFile | undefined
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      for (const [, part] of parts) {
-        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
-        assembled ??= YencFile.fromSegment(segment)
-        assembled.add(segment)
+    const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
+    const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
+    let releaseId = file.release_id
+    if (weak) {
+      const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
+      if (identity) {
+        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
+          { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
+        ])
+        releaseId = ids.get(identity.key) ?? releaseId
       }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
     }
-    return assembled!.getBuffer()
+    await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
+    if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
+    return yencName
   }
 
   /**
-   * Downloads a small archive that lists NZBs and imports each of them
-   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
-   * uploads by other posters.
+   * Names a niconico capture's release after the video, while it still has
+   * the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
+   * the search text.
    */
-  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
-    const archive = await this.readWhole(first, file.message_ids)
-    const info = archive && parseRarHeaders(archive)
-    if (!archive || !info) return []
-    const entries = info.entries.filter(isNzbEntry)
-    const results: NzbImportResult[] = []
-    for (const entry of entries) {
-      const xml = await extractEntry(archive, entry)
-      if (!xml) continue
-      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
-        entryName: entry.name,
-        moveContainer: entries.length === 1,
-      })
-      if (imported) results.push(imported)
-    }
-    return results
+  private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
+    if (!file.release_id) return undefined
+    const release = await db.from('releases').where('id', file.release_id).first()
+    if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
+    const name = nicovideoReleaseName(info, id)
+    await db
+      .from('releases')
+      .where('id', release.id)
+      .update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
+    return name
   }
 
-  async inspect(fileId: number): Promise<ContentResult | null> {
+  /**
+   * The database half before a probe (CONCEPT.md D41): what to inspect for
+   * a file, or null when there's nothing to do. A later 7z volume becomes its
+   * archive's first volume, once all volumes are in.
+   */
+  async prepare(fileId: number): Promise<InspectTask | null> {
     const file = await db.from('files').where('id', fileId).first()
-    const binary = await db.from('binaries').where('file_id', fileId).first()
     if (!file || !this.isCandidate(file.filename)) return null
-    const first = file.message_ids?.['1']
-    if (!first?.id) return null
+    // Stored before its signature was found to be garbage (D42): not worth an article.
+    if (await GarbageFilter.skipProbe(file)) return null
+    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
+      const volumes = await SevenZipService.volumes(file)
+      if (!volumes || (await FileStore.details(Number(volumes[0].id))).archive) return null
+      return this.prepare(Number(volumes[0].id))
+    }
+    const segments = segmentsOf(file)
+    if (!segments[0]?.id) return null
+    // A 7z listing reads the last volume's tail (sniffing decides whether it's 7z: random names too).
+    const volumes = await SevenZipService.volumes(file)
+    const last = volumes && volumes.length > 1 ? segmentsOf(volumes[volumes.length - 1]) : segments
+    return {
+      kind: 'inspect',
+      fileId: Number(file.id),
+      name: String(file.real_name ?? file.filename),
+      segments: taskSegments(segments),
+      sevenZip: volumes ? { volumes: volumes.length, last: taskSegments(last, true) } : null,
+    }
+  }
 
-    const conn = await NntpService.acquire()
-    let failed = false
-    let body: Buffer
+  /**
+   * The database half after a probe: names, groupings, listings and flags from
+   * what it found. Then the release is deleted if it turned out to be garbage (D42).
+   */
+  async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
+    let result: ContentResult | undefined
+    // Another probe deleted a release this one writes to as garbage meanwhile (LOG.md 2026-10-08 (7)):
+    // the file's own release (nothing left to do), or one its par2 set joins. Again once that
+    // deletion has committed: the set then finds the release's files gone and makes its own.
+    for (const wait of [0, 1000, 3000]) {
+      if (wait) await new Promise((resolve) => setTimeout(resolve, wait))
       try {
-      body = await NntpService.fetchBody(conn, first.id)
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
+        result = await this.applyProbe(task, probe)
+        break
+      } catch (error: any) {
+        if (error.code !== '23503' || wait === 3000) throw error
+        if (!(await db.from('files').where('id', task.fileId).select('id').first())) {
+          return { named: 0, contentType: probe.contentType, garbage: 'deleted meanwhile' }
         }
-    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
-    let segment
-    try {
-      segment = decodeSegment(body)
-    } catch (error) {
-      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
-      segment = decodeSegment(body, { lenientSize: true })
       }
-    const size = segment.meta.begin.size
-    const contentType = sniffContentType(segment.data)
-    await db.from('files').where('id', fileId).update({ content_type: contentType })
+    }
+    if (!result) throw new Error(`Applying the probe of file ${task.fileId} gave no result`)
+    result.garbage = (await GarbageFilter.afterProbe(task.fileId, probe.contentType)) ?? undefined
+    return result
+  }
+
+  private async applyProbe(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
+    const result: ContentResult = { named: 0, contentType: probe.contentType }
+    const file = await db.from('files').where('id', task.fileId).first()
+    if (!file) return result
+    const binary = await db.from('binaries').where('file_id', task.fileId).first()
+    const { contentType } = probe
+    await FileStore.setDetails(Number(file.id), { content_type: contentType })
+    if (contentType === 'missing') {
+      // Nothing to read; the binary needn't wait for inspection any more.
+      if (binary) await db.from('binaries').where('id', binary.id).update({ awaiting_inspection: null })
+      return result
+    }
 
     // Fingerprint, so par2 sets (now or later) can name this file exactly.
     if (binary) {
@@ -136,14 +198,16 @@ class ContentService {
         .from('binaries')
         .where('id', binary.id)
         .update({
-          size: db.raw('coalesce(size, ?)', [size]),
-          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
+          size: db.raw('coalesce(size, ?)', [probe.size]),
+          md5_16k: db.raw('coalesce(md5_16k, ?)', [probe.md5_16k]),
+          // Inspected: the GC may delete it again by its usual rules (LOG.md 2026-10-06 (11)).
+          awaiting_inspection: null,
         })
     }
 
-    const result: ContentResult = { named: 0, contentType }
-    if (contentType === 'nzbe' || contentType === 'rclone') {
-      // An encrypted NZB, or rclone crypt (names and data encrypted): nothing
+    result.yencName = await this.nameFromYenc(file, binary, probe.yencName ?? '')
+    if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
+      // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
       // to read, and useless without the key.
       const releaseId = file.release_id ?? binary?.release_id
       if (releaseId) {
@@ -151,37 +215,107 @@ class ContentService {
       }
       return result
     }
-    if (contentType === 'nzb') {
-      const xml = await this.readWhole(segment.data, file.message_ids)
-      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
+    const nicovideo = NICOVIDEO_FILE.exec(task.name)
+    if (nicovideo) {
+      if (probe.nicovideo) result.nicovideo = await this.nameNicovideo(file, probe.nicovideo, nicovideo[1])
       return result
     }
-    const set = await Par2Service.read(segment.data, file.message_ids)
-    if (set && binary) {
-      result.par2 = await Par2Service.attach(binary, set)
-    } else {
-      const info = parseRarHeaders(segment.data)
-      if (info) {
-        result.archive = await ArchiveService.record(file, binary, info)
-        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
+    if (/\.sfv$/i.test(task.name)) {
+      if (typeof probe.text === 'string') result.sfvMoved = (await SfvService.attach(file, probe.text)) ?? undefined
+      return result
+    }
+    if (contentType === 'nzb') {
+      if (typeof probe.text === 'string') result.nzb = (await NzbImportService.import(file, probe.text)) ?? undefined
+      return result
+    }
+    if (probe.par2 && binary) {
+      result.par2 = await Par2Service.attach(binary, probe.par2)
+    } else if (probe.archive && probe.archive.format === '7z') {
+      result.archive = await ArchiveService.record(file, binary, probe.archive)
+    } else if (probe.archive) {
+      result.archive = await ArchiveService.record(file, binary, probe.archive)
+      // The rest of the listing, later and at low priority (D40).
+      await ArchiveWalkService.queueAfterInspection(result.archive.releaseId ?? null, probe.archive)
+      if (probe.archiveNzbs?.length) {
+        result.archiveNzbs = []
+        for (const { entryName, xml } of probe.archiveNzbs) {
+          const imported = await NzbImportService.import(file, xml, { entryName, moveContainer: (probe.archiveNzbEntries ?? 0) === 1 })
+          if (imported) result.archiveNzbs.push(imported)
+        }
       }
     }
     if (binary) result.named = await NameService.resolve([binary.id])
     return result
   }
+
+  /** Inspects a file here: prepare, probe on the inspection pool's connections, apply. */
+  async inspect(fileId: number): Promise<ContentResult | null> {
+    const task = await this.prepare(fileId)
+    if (!task) return null
+    const probe = await probeInspection(task, fetchPooled, (message) => logger.warn(message))
+    return this.apply(task, probe)
+  }
 }
 
-/** NZBs that can be read from a single, unencrypted archive. */
-function hasNzbEntries(info: RarInfo): boolean {
-  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+/** A task's segment list: all of a small file's; of a big one, the first and the last few (a 7z tail). */
+function taskSegments(segments: Segment[], tailOnly = false): TaskSegment[] {
+  const list = segments.map(({ number, id, size }) => ({ number, id, size }))
+  if (list.length <= TASK_SEGMENTS) return list
+  return tailOnly ? list.slice(-TASK_SEGMENTS) : [list[0], ...list.slice(-TASK_SEGMENTS)]
 }
 
-function isNzbEntry(entry: RarEntry): boolean {
-  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+/** More than whole files (40 parts) and par2 files (20) are read; a 7z tail of 16 MB fits easily. */
+const TASK_SEGMENTS = 40
+
+/** One article body on a pooled connection. */
+async function fetchPooled(messageId: string): Promise<Buffer> {
+  const conn = await NntpService.acquire()
+  let failed = false
+  try {
+    return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
+  } catch (error: any) {
+    // A missing article (430) leaves the connection fine.
+    failed = !/^4\d\d/.test(String(error.message))
+    throw error
+  } finally {
+    NntpService.release(conn, failed)
+  }
 }
 
-/** Small posted files (NZBs) that are read completely. */
-const WHOLE_MAX_PARTS = 40
-const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+/**
+ * One line per finding, as worker:collection logs them (and the seed API
+ * for nodes' results).
+ */
+export function describeContent(filename: string, content: ContentResult | null): string[] {
+  const lines: string[] = []
+  if (content?.par2) {
+    const { par2 } = content
+    lines.push(`Par2 "${filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
+  } else if (content?.archive) {
+    const { info, releaseRenamed, grouped } = content.archive
+    const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
+      .filter(Boolean)
+      .join(', ')
+    lines.push(
+      `Archive "${filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
+        (grouped ? ', grouped into a release' : '') +
+        (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
+    )
+  }
+  if (content?.named) lines.push(`"${filename}" got its real name from par2.`)
+  if (content?.contentType === 'missing') lines.push(`"${filename}": the server no longer has its first article.`)
+  if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
+  if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
+  if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
+  if (content?.garbage) lines.push(`"${filename}": release deleted as garbage (${content.garbage === 'deleted meanwhile' ? 'by another probe' : `${content.garbage}, no name or password`}).`)
+  if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
+  if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
+  if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)
+  if (content?.nzb) {
+    const n = content.nzb
+    lines.push(`NZB "${filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
+  }
+  return lines
+}
 
 export default new ContentService()

app/services/FetchRangeService.ts

@@ -1,6 +1,7 @@
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import QueueService from '#services/QueueService'
+import indexerConfig from '#config/indexer'
 
 /** A failed range is retried after this long... */
 const REQUEUE_AFTER_MS = 5 * 60 * 1000
@@ -20,21 +21,22 @@ interface Group {
  * retried instead of silently skipped. See CONCEPT.md (D18).
  */
 class FetchRangeService {
-  async queue(group: Group, startId: bigint, endId: bigint) {
+  /** Queues a range. A backfill range is header-only and waits behind live ones (CONCEPT.md D32). */
+  async queue(group: Group, startId: bigint, endId: bigint, { backfill = false } = {}) {
     const now = new Date()
     const [row] = await db
       .table('fetch_ranges')
-      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), created_at: now, updated_at: now })
+      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), backfill, created_at: now, updated_at: now })
       .returning(['id', 'requeues'])
-    await this.addJob(group.name, Number(row.id), 0, startId, endId)
+    await this.addJob(group.name, Number(row.id), 0, startId, endId, backfill)
   }
 
-  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint) {
+  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint, backfill: boolean) {
     await QueueService.nntpFetchQueue.add(
       'fetch-headers',
-      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId },
+      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId, ...(backfill ? { backfill: true } : {}) },
       // The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
-      { jobId: this.jobId(rangeId, requeues) }
+      { jobId: this.jobId(rangeId, requeues), ...(backfill ? { priority: indexerConfig.backfill.priority } : {}) }
     )
   }
 
@@ -75,7 +77,7 @@ class FetchRangeService {
         .from('fetch_ranges')
         .where('id', range.id)
         .update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
-      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id))
+      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id), Boolean(range.backfill))
       requeued++
       logger.warn(
         `Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
@@ -85,13 +87,14 @@ class FetchRangeService {
   }
 
   /**
-   * Moves `groups.last_indexed_id` up to just below the first range that is
-   * not done yet, and forgets done ranges. Returns the new watermark.
+   * Moves `groups.last_indexed_id` up to just below the first live range
+   * that is not done yet (backfill ranges lie below it and don't count), and
+   * forgets done ranges. Returns the new watermark.
    */
   async advanceWatermark(group: Group): Promise<bigint> {
     const { rows } = await db.rawQuery(
       `UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
-         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done'),
+         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done' AND NOT backfill),
          g.last_queued_id))
        WHERE g.id = ? RETURNING last_indexed_id`,
       [group.id]

app/services/FileKeys.ts

@@ -0,0 +1,60 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** files ids per slice: a few seconds of reads from the HDD. */
+const SLICE = 100_000
+
+/**
+ * The file_keys filter (LOG.md 2026-10-06 (7)): a 64-bit hash of every
+ * file_key. The insert trigger adds new keys; `fill` copies the keys of
+ * files from before, in id order and a slice at a time, and marks the
+ * filter complete when it reaches the files that existed when it started
+ * (everything after that has been added by the trigger, which was live
+ * first). FileStore uses the filter only once it is complete.
+ */
+class FileKeys {
+  private completeSince = 0
+
+  /** Whether every file's key is in file_keys; cached once true. */
+  async complete(): Promise<boolean> {
+    if (this.completeSince) return true
+    const row = await db.from('file_keys_fill').where('id', 1).first()
+    if (row?.complete) this.completeSince = Date.now()
+    return Boolean(row?.complete)
+  }
+
+  /** Forgets the cached state (tests). */
+  forget() {
+    this.completeSince = 0
+  }
+
+  /** Copies keys until `budgetMs` has passed or the fill is done; returns the files read and whether it's complete. */
+  async fill(budgetMs: number): Promise<{ read: number; complete: boolean; nextId: number; untilId: number }> {
+    const started = Date.now()
+    let state = await db.from('file_keys_fill').where('id', 1).first()
+    if (!state) return { read: 0, complete: false, nextId: 0, untilId: 0 }
+    if (state.until_id === null) {
+      // Everything inserted from here on goes through the trigger (live since the migration).
+      const [{ max }] = await db.from('files').max('id as max')
+      await db.from('file_keys_fill').where('id', 1).update({ until_id: Number(max ?? 0), updated_at: new Date() })
+      state = await db.from('file_keys_fill').where('id', 1).first()
+    }
+    let next = Number(state.next_id)
+    const until = Number(state.until_id)
+    let read = 0
+    while (next <= until && Date.now() - started < budgetMs) {
+      const to = Math.min(until, next + SLICE - 1)
+      const result = await db.rawQuery(
+        `INSERT INTO file_keys (key_hash)
+         SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE id BETWEEN ? AND ? AND file_key IS NOT NULL
+         ON CONFLICT DO NOTHING`,
+        [next, to]
+      )
+      read += Number(result.rowCount ?? 0)
+      next = to + 1
+      await db.from('file_keys_fill').where('id', 1).update({ next_id: next, complete: next > until, updated_at: new Date() })
+    }
+    return { read, complete: next > until, nextId: next, untilId: until }
+  }
+}
+
+export default new FileKeys()

app/services/FilePartitions.ts

@@ -0,0 +1,50 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Ids per partition (as the migration that partitioned files: …150). */
+export const FILE_PARTITION_SIZE = 4_000_000
+/**
+ * Ids kept covered beyond the newest file: four partitions (16M ids, about a day of
+ * seeding), so a few sweeps that can't create one are harmless.
+ */
+const AHEAD = 4 * FILE_PARTITION_SIZE
+
+/**
+ * `files` is partitioned by id (CONCEPT.md D39). An insert with an id no
+ * partition covers fails, so gc:scheduler calls ensure() every sweep to keep
+ * four partitions ready ahead of the newest id. Each new partition gets its
+ * own unique index on file_key (FileStore.insert checks the older ones),
+ * in the parent's tablespace (the HDD on homelab).
+ */
+class FilePartitions {
+  async ensure(): Promise<string[]> {
+    const created: string[] = []
+    while (true) {
+      // The sequence, not max(id): cheaper, and it counts ids handed out but not committed yet.
+      const { rows } = await db.rawQuery(
+        `SELECT (SELECT last_value FROM files_id_seq) AS max_id,
+                -- ''{0,1}, not ''?: knex reads ? as a binding.
+                max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
+         FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
+         WHERE i.inhparent = 'files'::regclass`
+      )
+      const maxId = Number(rows[0].max_id)
+      const upper = Number(rows[0].upper)
+      if (upper - maxId >= AHEAD) return created
+      const name = `files_p${upper}`
+      // CREATE … PARTITION OF locks files and its partitions one by one, which deadlocked with
+      // file inserts (11 failed sweeps on 2026-10-07). Taking the parent's lock first, in one
+      // statement, can only wait; after 10 s it gives up until the next sweep.
+      await db.transaction(async (trx) => {
+        await trx.rawQuery(`SET LOCAL lock_timeout = '10s'`)
+        await trx.rawQuery(`LOCK TABLE files IN ACCESS EXCLUSIVE MODE`)
+        await trx.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`)
+        await trx.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
+      })
+      logger.info(`files: partition ${name} created (ids ${upper}–${upper + FILE_PARTITION_SIZE - 1}).`)
+      created.push(name)
+    }
+  }
+}
+
+export default new FilePartitions()

app/services/FileStore.ts

@@ -0,0 +1,138 @@
+import db from '@adonisjs/lucid/services/db'
+import FileKeys from '#services/FileKeys'
+import type { QueryClientContract } from '@adonisjs/lucid/types/database'
+import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
+
+/**
+ * The only code that knows how `files` keeps its segment lists (CONCEPT.md
+ * D31): compressed in `files.segments`, with the first segment's message-id
+ * in the indexed `first_message_id` for lookups. Read them with
+ * `segmentsOf(file)`; write new files with `FileStore.insert`.
+ */
+
+/** A new `files` row, with its segments as a list. */
+export type NewFile = Record<string, unknown> & { segments: Segment[] }
+
+const unpacked = new WeakMap<object, Segment[]>()
+
+/** The segments of a `files` row (a row from `db.from('files')`), in order. */
+export function segmentsOf(file: { segments?: Buffer | Segment[] | null }): Segment[] {
+  const { segments } = file
+  if (!segments) return []
+  if (Array.isArray(segments)) return segments
+  let list = unpacked.get(file)
+  if (!list) {
+    list = unpackSegments(segments)
+    unpacked.set(file, list)
+  }
+  return list
+}
+
+/** Stored form of a segment list, for the File model's column. */
+export function storedSegments(segments: Segment[]) {
+  return packSegments(segments)
+}
+
+export function loadedSegments(stored: Buffer | null): Segment[] {
+  return stored ? unpackSegments(stored) : []
+}
+
+/** What inspection learns about a file, kept beside `files` (LOG.md 2026-10-08 (14)). */
+export interface FileDetails {
+  content_type?: string | null
+  archive?: unknown
+}
+
+const DETAIL_COLUMNS = ['content_type', 'archive'] as const
+
+/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
+export const insertLimits = { rows: 2000 }
+
+class FileStore {
+  /** Sets some of a file's details, leaving the others as they are. */
+  async setDetails(fileId: number, details: FileDetails, client: Pick<QueryClientContract, 'rawQuery'> = db) {
+    const columns = DETAIL_COLUMNS.filter((c) => details[c] !== undefined)
+    if (columns.length === 0) return
+    const values = columns.map((c) => (c === 'archive' && details[c] !== null ? JSON.stringify(details[c]) : details[c]))
+    await client.rawQuery(
+      `INSERT INTO file_details (file_id, ${columns.join(', ')}) VALUES (?, ${columns.map(() => '?').join(', ')})
+       ON CONFLICT (file_id) DO UPDATE SET ${columns.map((c) => `${c} = excluded.${c}`).join(', ')}`,
+      [fileId, ...(values as any[])]
+    )
+  }
+
+  /** A file's details (empty when it has none). */
+  async details(fileId: number): Promise<FileDetails> {
+    return (await db.from('file_details').where('file_id', fileId).select('content_type', 'archive').first()) ?? {}
+  }
+
+  /**
+   * Inserts files, skipping those whose `file_key` exists. Returns the rows
+   * that were inserted (id, filename, file_key). `files` is partitioned by id
+   * (CONCEPT.md D39) and each partition has its own unique file_key index:
+   * once file_keys (a 64-bit hash of every key) is complete, it says which keys
+   * may exist, and only those are looked up in the partitions; two
+   * writers adding the same new key at once meet in the newest partition's index.
+   */
+  async insert(
+    files: NewFile[],
+    client: Pick<QueryClientContract, 'knexQuery' | 'rawQuery'> = db
+  ): Promise<{ id: number; filename: string; file_key: string | null }[]> {
+    if (files.length === 0) return []
+    const keys = [...new Set(files.map((file) => file.file_key).filter((key): key is string => typeof key === 'string'))]
+    const known = new Set<string>()
+    const filtered = await FileKeys.complete()
+    for (let i = 0; i < keys.length; i += 5000) {
+      const chunk = keys.slice(i, i + 5000)
+      if (!filtered) {
+        const found = await client.knexQuery().from('files').whereIn('file_key', chunk).select('file_key')
+        for (const row of found) known.add(row.file_key)
+        continue
+      }
+      const { rows } = await client.rawQuery(
+        `SELECT k.key FROM unnest(?::text[]) AS k(key)
+         WHERE EXISTS (SELECT 1 FROM file_keys f WHERE f.key_hash = hashtextextended(k.key, 0))`,
+        [chunk]
+      )
+      const maybe = rows.map((row: { key: string }) => row.key)
+      if (maybe.length === 0) continue
+      const found = await client.knexQuery().from('files').whereIn('file_key', maybe).select('file_key')
+      for (const row of found) known.add(row.file_key)
+    }
+    const seen = new Set<string>()
+    const rows = files
+      .filter((file) => {
+        const key = file.file_key as string | null | undefined
+        if (!key) return true
+        if (known.has(key) || seen.has(key)) return false
+        seen.add(key)
+        return true
+      })
+      .map(({ segments, ...file }) => {
+        const sorted = [...segments].sort((a, b) => a.number - b.number)
+        return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
+      })
+    if (rows.length === 0) return []
+    // Details given with a new file (e.g. a known content type) go to file_details.
+    const details = rows.map((row: Record<string, unknown>) => {
+      const d: FileDetails = {}
+      for (const c of DETAIL_COLUMNS) if (c in row) (d as any)[c] = row[c]
+      for (const c of DETAIL_COLUMNS) delete row[c]
+      return d
+    })
+    // A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
+    const inserted: { id: number; filename: string; file_key: string | null }[] = []
+    for (let i = 0; i < rows.length; i += insertLimits.rows) {
+      const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename', 'file_key'])
+      for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename, file_key: row.file_key ?? null })
+    }
+    const detailsByKey = new Map(rows.map((row: Record<string, unknown>, i: number) => [row.file_key ?? row.filename, details[i]]))
+    for (const row of inserted) {
+      const d = detailsByKey.get(row.file_key ?? row.filename)
+      if (d && Object.keys(d).length > 0) await this.setDetails(row.id, d, client)
+    }
+    return inserted
+  }
+}
+
+export default new FileStore()

app/services/GarbageFilter.ts

@@ -0,0 +1,217 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import { looksRandom } from '#services/ArchiveService'
+import { fileStem } from '#lib/release/index'
+import { explored, signatureGroup, signatureKey, uploadTool } from '#lib/garbage/index'
+
+/** What the filter needs to know about an upload before it is stored. */
+export interface Upload {
+  /** The posted file name. */
+  name: string
+  /** Any of its articles' Message-IDs (one tool posts all of a file). */
+  messageId: string | null | undefined
+  groups: unknown
+  /** Its release's key (the file key when it has none): kept or dropped as a whole. */
+  releaseKey: string
+}
+
+/** Contents that can't name a random-named release without a key (LOG.md 2026-10-08 (5)). */
+const CONTAINERS = ['rclone', 'nzbe', 'yencbin']
+
+/** How long the dropped signatures are cached per process. */
+const CACHE_MS = 5 * 60 * 1000
+
+export function isRandomName(name: string) {
+  return looksRandom(fileStem(name))
+}
+
+/**
+ * Garbage: random-named uploads that never get a real name and have no
+ * known password (CONCEPT.md D42). Two filters:
+ * - before storing (seeding, ingest): random-named uploads whose signature
+ *   (posting tool, first group) is dropped, or whose release was dropped
+ *   after its probe, aren't stored;
+ * - after a probe: a random-named release whose probes are all done, that
+ *   is still without a real name, password, names from its files or spot,
+ *   and is encrypted, a container or missing its first article, is deleted.
+ * The dropped signatures are learned from a sample of probed releases
+ * (learn(), gc:scheduler).
+ */
+class GarbageFilter {
+  private dropped: Set<string> | null = null
+  private loadedAt = 0
+
+  /** The dropped signatures, cached for CACHE_MS. */
+  async droppedSignatures(): Promise<Set<string>> {
+    if (this.dropped && Date.now() - this.loadedAt < CACHE_MS) return this.dropped
+    const rows = await db.from('upload_signatures').where('dropped', true).select('tool', 'group_name')
+    this.dropped = new Set(rows.map((row) => signatureKey(row.tool, row.group_name)))
+    this.loadedAt = Date.now()
+    return this.dropped
+  }
+
+  /** Forgets the cached signatures (after learn(), and in tests). */
+  reset() {
+    this.dropped = null
+  }
+
+  /** Whether a random-named upload's signature is dropped and it isn't in the explored share. */
+  private droppedBySignature(upload: Upload, signatures: Set<string>) {
+    if (signatures.size === 0) return false
+    if (!signatures.has(signatureKey(uploadTool(upload.messageId), signatureGroup(upload.groups)))) return false
+    return !explored(upload.releaseKey, indexerConfig.garbage.explorePercent)
+  }
+
+  /**
+   * The items worth storing: drops random-named uploads of dropped signatures
+   * (but explorePercent of their releases) and of dropped releases.
+   */
+  async keep<T>(items: T[], upload: (item: T) => Upload): Promise<{ kept: T[]; dropped: number }> {
+    if (!indexerConfig.garbage.enabled || items.length === 0) return { kept: items, dropped: 0 }
+    const signatures = await this.droppedSignatures()
+    // Per release, kept as a whole when any of its files has a readable name. A file is
+    // looked at once: an ingest batch has many articles of each.
+    const verdicts = new Map<string, 'keep' | 'drop' | 'check'>()
+    const seen = new Set<string>()
+    for (const item of items) {
+      const u = upload(item)
+      if (verdicts.get(u.releaseKey) === 'keep' || seen.has(`${u.releaseKey}\n${u.name}`)) continue
+      seen.add(`${u.releaseKey}\n${u.name}`)
+      if (!isRandomName(u.name)) verdicts.set(u.releaseKey, 'keep')
+      else if (!verdicts.has(u.releaseKey)) verdicts.set(u.releaseKey, this.droppedBySignature(u, signatures) ? 'drop' : 'check')
+    }
+    const check = [...verdicts].filter(([, v]) => v === 'check').map(([key]) => key)
+    for (let i = 0; i < check.length; i += 5000) {
+      const rows = await db.from('dropped_releases').whereIn('release_key', check.slice(i, i + 5000)).select('release_key')
+      for (const row of rows) verdicts.set(row.release_key, 'drop')
+    }
+    const kept = items.filter((item) => verdicts.get(upload(item).releaseKey) !== 'drop')
+    return { kept, dropped: items.length - kept.length }
+  }
+
+  /**
+   * Whether a stored file's probe can be skipped: a random-named file of a
+   * dropped signature, outside the explored share (files stored before its
+   * signature was dropped). Its binary stops waiting for inspection.
+   */
+  async skipProbe(file: { id: number | string; filename: string; real_name?: string | null; first_message_id?: string | null; groups: unknown; release_id?: number | string | null }) {
+    if (!indexerConfig.garbage.enabled || file.real_name || !isRandomName(file.filename)) return false
+    const signatures = await this.droppedSignatures()
+    if (!signatures.has(signatureKey(uploadTool(file.first_message_id), signatureGroup(file.groups)))) return false
+    const release = file.release_id ? await db.from('releases').where('id', file.release_id).select('release_key').first() : null
+    if (explored(release?.release_key ?? String(file.id), indexerConfig.garbage.explorePercent)) return false
+    await db.from('binaries').where('file_id', file.id).where('awaiting_inspection', true).update({ awaiting_inspection: null })
+    return true
+  }
+
+  /**
+   * After a file's probe has been applied: notes its release for learning
+   * (a sample) and deletes the release when it is garbage. Never throws:
+   * the probe's result is stored already. Returns why the release was
+   * deleted, or null.
+   */
+  async afterProbe(fileId: number, contentType: string | null): Promise<string | null> {
+    try {
+      const file = await db.from('files').where('id', fileId).select('id', 'filename', 'real_name', 'first_message_id', 'groups', 'release_id').first()
+      if (!file?.release_id || !isRandomName(String(file.filename))) return null
+      const releaseId = Number(file.release_id)
+      if (releaseId % indexerConfig.garbage.probeSample === 0) {
+        await db.rawQuery(`INSERT INTO signature_probes (release_id, tool, group_name) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [
+          releaseId,
+          uploadTool(file.first_message_id),
+          signatureGroup(file.groups),
+        ])
+      }
+      if (!indexerConfig.garbage.enabled) return null
+      return await this.dropIfGarbage(releaseId, contentType)
+    } catch (error: any) {
+      logger.warn(`Garbage check after probing file ${fileId}: ${error.message}`)
+      return null
+    }
+  }
+
+  private async dropIfGarbage(releaseId: number, contentType: string | null): Promise<string | null> {
+    const { rows } = await db.rawQuery(
+      `SELECT r.release_key, r.container, r.encrypted,
+         is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
+           OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id) AS named,
+         EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id AND b.awaiting_inspection) AS waiting
+       FROM releases r WHERE r.id = ?`,
+      [releaseId]
+    )
+    const release = rows[0]
+    if (!release || release.named || release.waiting) return null
+    const reason = CONTAINERS.includes(release.container) ? release.container : release.encrypted === true ? 'encrypted' : contentType === 'missing' ? 'missing' : null
+    if (!reason) return null
+    await db.transaction(async (trx) => {
+      await trx.rawQuery(`INSERT INTO dropped_releases (release_key, reason) VALUES (?, ?) ON CONFLICT DO NOTHING`, [release.release_key, reason])
+      await trx.rawQuery(`DELETE FROM binaries WHERE release_id = ?`, [releaseId])
+      await trx.rawQuery(`DELETE FROM files WHERE release_id = ?`, [releaseId])
+      await trx.rawQuery(`DELETE FROM releases WHERE id = ?`, [releaseId])
+    })
+    return reason
+  }
+
+  /**
+   * Learns the signatures from the probed releases of the last windowDays:
+   * dropped when at least minProbes are known and at most maxNamedRate got a
+   * real name, password, names from their files or a spot. A signature with
+   * fewer probes keeps its verdict (a dropped one only gets explored probes).
+   * Releases deleted as garbage count as unnamed. Returns the signatures
+   * whose verdict changed.
+   */
+  async learn(): Promise<{ signatures: number; dropped: number; changed: string[] }> {
+    const { minProbes, maxNamedRate, windowDays } = indexerConfig.garbage
+    await db.rawQuery(`DELETE FROM signature_probes WHERE probed_at < now() - make_interval(days => ?)`, [windowDays])
+    const before = new Map((await db.from('upload_signatures').select('tool', 'group_name', 'dropped')).map((row) => [signatureKey(row.tool, row.group_name), row.dropped]))
+    await db.rawQuery(
+      `INSERT INTO upload_signatures (tool, group_name, probed, named, dropped, updated_at)
+       SELECT tool, group_name, probed, named, probed >= ?::int AND named <= probed * ?::float8, now()
+       FROM (
+         SELECT p.tool, p.group_name, count(*)::int AS probed,
+           (count(*) FILTER (WHERE is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
+              OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)))::int AS named
+         FROM signature_probes p LEFT JOIN releases r ON r.id = p.release_id
+         GROUP BY p.tool, p.group_name
+       ) s
+       ON CONFLICT (tool, group_name) DO UPDATE SET
+         probed = excluded.probed, named = excluded.named, updated_at = now(),
+         dropped = CASE WHEN excluded.probed >= ?::int THEN excluded.dropped ELSE upload_signatures.dropped END`,
+      [minProbes, maxNamedRate, minProbes]
+    )
+    const after = await db.from('upload_signatures').select('tool', 'group_name', 'dropped', 'probed', 'named')
+    const changed = after
+      .filter((row) => (before.get(signatureKey(row.tool, row.group_name)) ?? false) !== row.dropped)
+      .map((row) => `${row.tool} ${row.group_name || '(no group)'}: ${row.dropped ? 'dropped' : 'kept'} (${row.named}/${row.probed} named)`)
+    this.reset()
+    return { signatures: after.length, dropped: after.filter((row) => row.dropped).length, changed }
+  }
+
+  /**
+   * Fills signature_probes from a sample of files probed before it existed
+   * (garbage:learn --seed): random-named files with a content type, from
+   * TABLESAMPLE SYSTEM (percent) of files. Returns the releases added.
+   */
+  async seed(percent: number): Promise<number> {
+    const { rows } = await db.rawQuery(
+      `SELECT DISTINCT ON (f.release_id) f.release_id, f.first_message_id, f.groups
+       FROM files f TABLESAMPLE SYSTEM (?) JOIN file_details d ON d.file_id = f.id
+       WHERE d.content_type IS NOT NULL AND f.release_id IS NOT NULL AND NOT is_readable_name(f.filename)`,
+      [percent]
+    )
+    let added = 0
+    for (let i = 0; i < rows.length; i += 2000) {
+      const chunk = rows.slice(i, i + 2000)
+      const result = await db.rawQuery(
+        `INSERT INTO signature_probes (release_id, tool, group_name)
+         SELECT * FROM unnest(?::bigint[], ?::text[], ?::text[]) ON CONFLICT DO NOTHING`,
+        [chunk.map((row: any) => row.release_id), chunk.map((row: any) => uploadTool(row.first_message_id)), chunk.map((row: any) => signatureGroup(row.groups))]
+      )
+      added += result.rowCount ?? 0
+    }
+    return added
+  }
+}
+
+export default new GarbageFilter()

app/services/IndexMaintenance.ts

@@ -0,0 +1,85 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Only indexes at least this big are worth checking. */
+const MIN_BYTES = 50 * 1024 * 1024
+/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
+const MAX_LEAF_DENSITY = 50
+
+export interface Rebuilt {
+  index: string
+  density: number
+  before: number
+  after: number
+}
+
+/**
+ * GC deletes millions of rows (single-article binaries, copied parts, empty
+ * releases). Vacuum makes their index entries reusable but doesn't compact
+ * the index, so B-trees on the churning tables end up 10–20% full (LOG.md
+ * 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
+ * D33). Needs the pgstattuple extension.
+ */
+class IndexMaintenance {
+  async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
+    const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
+    if (extension.length === 0) {
+      logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
+      return []
+    }
+    if (!dryRun) await this.dropLeftovers()
+
+    const { rows: candidates } = await db.rawQuery(
+      `SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
+       FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
+       JOIN pg_namespace n ON n.oid = c.relnamespace
+       WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
+       ORDER BY 2 DESC`,
+      [MIN_BYTES]
+    )
+    const rebuilt: Rebuilt[] = []
+    for (const { index, bytes } of candidates) {
+      const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
+      const density = Number(rows[0].avg_leaf_density)
+      if (density >= MAX_LEAF_DENSITY) continue
+      if (!dryRun && !(await this.reindex(index))) continue
+      const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
+      const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
+      rebuilt.push(result)
+      logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
+    }
+    return rebuilt
+  }
+
+  /**
+   * REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
+   * binaries during a backfill). Its half-built copy is dropped and it is
+   * tried once more; a second failure skips the index until the next run.
+   */
+  /** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
+  private async dropLeftovers(index = '') {
+    const { rows } = await db.rawQuery(
+      `SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
+       WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
+      [index]
+    )
+    for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
+  }
+
+  private async reindex(index: string): Promise<boolean> {
+    for (let attempt = 1; attempt <= 2; attempt++) {
+      try {
+        await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
+        return true
+      } catch (error: any) {
+        logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
+        await this.dropLeftovers(index)
+      }
+    }
+    return false
+  }
+}
+
+const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`
+
+export default new IndexMaintenance()

app/services/IngestService.ts

@@ -4,43 +4,11 @@ import type { QueryClientContract } from '@adonisjs/lucid/types/database'
 import type { Knex } from 'knex'
 import QueueService from '#services/QueueService'
 import NameService from '#services/NameService'
-import { parseXrefGroups } from '#lib/nntp/xref'
-import { fileKey, parseSubject, weakFileKey } from '#lib/subject/index'
-import { releaseIdentity, type ReleaseIdentity } from '#lib/release/index'
-import { inferTotalParts, type YencMeta } from '#lib/yenc/index'
+import GarbageFilter from '#services/GarbageFilter'
+import type { IngestSegment } from '#lib/ingest/segment'
 
-/** One segment, with the identity of the file it belongs to already resolved. */
-export interface IngestSegment {
-  fileKey: string
-  poster: string
-  /** Subject without segment counter. */
-  subject: string
-  name: string
-  totalParts: number
-  number: number
-  /** Without angle brackets. */
-  messageId: string
-  bytes: number
-  articleNumber?: bigint | number
-  postedAt?: Date
-  groups: string[]
-  /**
-   * The identity comes from the yEnc name only (obfuscated posts). Such names
-   * can be shared by several equal-sized files of one post, so a second
-   * article for the same part number marks the binary as conflicted.
-   */
-  weakIdentity?: boolean
-  /** Release (download) this file belongs to, when it can be determined. */
-  release?: ReleaseIdentity | null
-  /** Exact file size from =ybegin (body-derived segments only). */
-  size?: number
-  /** yEnc CRCs (body-derived segments only), to verify split volumes. */
-  pcrc32?: string
-  partSize?: number
-  fileCrc32?: string
-  /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
-  md5_16k?: string
-}
+// Header parsing is pure and lives in lib, so seeding nodes need no database (CONCEPT.md D37).
+export { segmentFromHeader, segmentFromYenc, type IngestSegment, type OverviewHeader } from '#lib/ingest/segment'
 
 export interface CompletedBinary {
   id: number
@@ -118,6 +86,9 @@ async function upsertKeyed(
     }
   }
 }
+/** Completed binaries per worker:file job. */
+const FILE_BATCH = 100
+
 /** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */
 export const PAR2_SIZE_ONLY_MIN = 1024 * 1024
 /** A par2 set and its files are posted together; don't match across days. */
@@ -134,81 +105,20 @@ function validDate(date?: Date) {
   return date && !Number.isNaN(date.getTime()) ? date : null
 }
 
-/** An XOVER overview entry as returned by nntp-js. */
-export type OverviewHeader = Record<string, string>
-
-function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
-  const xrefGroups = parseXrefGroups(header.xref)
-  return {
-    poster: header.from ?? '',
-    messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
-    bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
-    articleNumber,
-    postedAt: header.date ? new Date(header.date) : undefined,
-    groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
-  }
-}
-
-/** Builds a segment from the subject line; null when the subject is obfuscated. */
-export function segmentFromHeader(
-  header: OverviewHeader,
-  groupName: string,
-  articleNumber?: bigint | number
-): IngestSegment | null {
-  const parsed = header.subject ? parseSubject(header.subject) : null
-  if (!parsed || !header['message-id']) return null
-  const base = common(header, groupName, articleNumber)
-  return {
-    ...base,
-    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
-    subject: parsed.base,
-    name: parsed.name,
-    totalParts: parsed.totalSegments,
-    number: parsed.segment,
-    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
-  }
-}
-
-/**
- * Builds a segment for an obfuscated post from the yEnc header in its body.
- * Subject and poster can be random per article, so the identity comes from
- * the yEnc header only (see weakFileKey).
- */
-export function segmentFromYenc(
-  header: OverviewHeader,
-  groupName: string,
-  meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
-  articleNumber?: bigint | number
-): IngestSegment | null {
-  const base = common(header, groupName, articleNumber)
-  const totalParts = inferTotalParts(meta)
-  const number = meta.begin.part ?? 1
-  if (!base.messageId || number < 1 || number > totalParts) return null
-  const subject = `"${meta.begin.name}" yEnc`
-  return {
-    ...base,
-    fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
-    subject,
-    name: meta.begin.name,
-    totalParts,
-    number,
-    weakIdentity: true,
-    release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
-    size: meta.begin.size,
-    // Single-part posts only carry crc32, which is then also the part's CRC.
-    pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
-    partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
-    fileCrc32: meta.end?.crc32,
-  }
-}
-
 /**
  * Stores segments in `binaries`/`parts` and queues every binary that became
  * complete exactly once. Idempotent: ingesting the same segments again is a
  * no-op. See CONCEPT.md (D12).
  */
 class IngestService {
-  async ingest(segments: IngestSegment[]): Promise<CompletedBinary[]> {
+  async ingest(batch: IngestSegment[]): Promise<CompletedBinary[]> {
+    // Garbage isn't stored at all: no release, binary or parts (CONCEPT.md D42).
+    const { kept: segments } = await GarbageFilter.keep(batch, (s) => ({
+      name: s.name,
+      messageId: s.messageId,
+      groups: s.groups,
+      releaseKey: s.release?.key ?? s.fileKey,
+    }))
     if (segments.length === 0) return []
 
     // Short statements, no enclosing transaction: every step is idempotent, and
@@ -260,6 +170,17 @@ class IngestService {
         for (const [key, id] of ids) idsByKey.set(key, id)
       }
 
+      // A binary that is a file already has its segment list in `files`; its
+      // header parts were dropped then. Crossposts in other groups, reposts
+      // and backfill ranges keep bringing the same articles, and inserting
+      // them again grew `parts` to 56M rows nobody reads (LOG.md 2026-10-04).
+      const done = new Set<number>()
+      const ids = [...idsByKey.values()]
+      for (const chunk of chunks(ids, BINARY_CHUNK)) {
+        const { rows } = await knex.raw('SELECT id FROM binaries WHERE id = ANY(?) AND file_id IS NOT NULL', [chunk])
+        for (const row of rows) done.add(Number(row.id))
+      }
+
       const partRows = segments
         .map((segment) => ({
           binary_id: idsByKey.get(segment.fileKey)!,
@@ -272,11 +193,34 @@ class IngestService {
           file_crc32: segment.fileCrc32 ?? null,
           md5_16k: segment.md5_16k ?? null,
         }))
-      const weakRows = partRows.filter((_, i) => segments[i].weakIdentity)
+      const weakRows = partRows.filter((row, i) => segments[i].weakIdentity && !(done.has(row.binary_id) && row.pcrc32 === null))
+      // Body-derived parts (with a per-part CRC) are kept, as worker:file does.
+      const newParts = partRows.filter((row) => !(done.has(row.binary_id) && row.pcrc32 === null))
 
-      partRows.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
-      for (const chunk of chunks(partRows, PART_CHUNK)) {
-        await knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore()
+      newParts.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
+      // Parts and the binaries' stored-part counts in one statement, so the count can't miss a
+      // part (LOG.md 2026-10-06 (7)); claimCompleted compares it instead of counting parts. NULL
+      // (binaries from before part_count) is counted once: the statement's own rows aren't in
+      // its snapshot, so they're added on top. Concurrent batches can lock binaries in a
+      // different order: a deadlock rolls the whole statement back, and it is retried.
+      for (const chunk of chunks(newParts, PART_CHUNK)) {
+        const insert = knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore().returning('binary_id').toSQL()
+        for (let attempt = 1; ; attempt++) {
+          try {
+            await knex.raw(
+              `WITH added AS (${insert.sql}),
+               counted AS (SELECT binary_id, count(*)::int AS n FROM added GROUP BY binary_id)
+               UPDATE binaries b SET part_count = CASE
+                 WHEN b.part_count IS NULL THEN (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) + c.n
+                 ELSE b.part_count + c.n END
+               FROM counted c WHERE b.id = c.binary_id`,
+              insert.bindings as any[]
+            )
+            break
+          } catch (error: any) {
+            if (error.code !== '40P01' || attempt >= 5) throw error
+          }
+        }
       }
 
       if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows)
@@ -360,18 +304,27 @@ class IngestService {
    */
   async matchPar2Releases(binaryIds: number[]) {
     if (binaryIds.length === 0) return
+    // Listed-once first (index-only on release_files (release_id, size): most candidate
+    // sizes are a set's equal-sized volumes), the release's date only for what's left;
+    // OFFSET 0 keeps that order. Was: every size match's date and count, 18 s per batch.
     await db.rawQuery(
       `UPDATE binaries b SET release_id = m.release_id
        FROM (
-         SELECT b2.id, min(rf.release_id) AS release_id
+         SELECT c.id, min(c.release_id) AS release_id
+         FROM (
+           SELECT b2.id, b2.posted_at, rf.release_id
            FROM binaries b2
            JOIN release_files rf ON rf.size = b2.size
-         JOIN releases r ON r.id = rf.release_id
            WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
-           AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ?
-           AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1
-         GROUP BY b2.id
-         HAVING count(DISTINCT rf.release_id) = 1
+             AND NOT EXISTS (
+               SELECT 1 FROM release_files x
+               WHERE x.release_id = rf.release_id AND x.size = rf.size AND x.par2_file_id <> rf.par2_file_id)
+           OFFSET 0
+         ) c
+         JOIN releases r ON r.id = c.release_id
+         WHERE abs(extract(epoch FROM (r.posted_at - c.posted_at))) < ?
+         GROUP BY c.id
+         HAVING count(DISTINCT c.release_id) = 1
        ) m
        WHERE b.id = m.id`,
       [binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
@@ -445,7 +398,7 @@ class IngestService {
     const result = await db.rawQuery(
       `UPDATE binaries b SET completed_at = now()
        WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL
-         AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) >= b.total_parts
+         AND coalesce(b.part_count, (SELECT count(*) FROM parts p WHERE p.binary_id = b.id)) >= b.total_parts
        RETURNING b.id, b.file_key`,
       [binaryIds]
     )
@@ -457,14 +410,14 @@ class IngestService {
 
   async enqueue(completed: CompletedBinary[]) {
     if (completed.length === 0) return
-    // jobId = file_key: BullMQ ignores an add() for an id that already exists.
-    await QueueService.fileQueue.addBulk(
-      completed.map(({ id, fileKey }) => ({
-        name: 'process-file',
-        data: { binaryId: id },
-        opts: { jobId: fileKey },
-      }))
-    )
+    // A job per FILE_BATCH binaries: worker:file stores them with a few set-based queries (LOG.md 2026-10-06 (7)).
+    // jobId from the first file_key: BullMQ ignores an add() for an id that already exists.
+    const jobs = []
+    for (let i = 0; i < completed.length; i += FILE_BATCH) {
+      const batch = completed.slice(i, i + FILE_BATCH)
+      jobs.push({ name: 'process-files', data: { binaryIds: batch.map((b) => b.id) }, opts: { jobId: `${batch[0].fileKey}-${batch.length}` } })
+    }
+    await QueueService.fileQueue.addBulk(jobs)
   }
 }
 

app/services/InspectionQueue.ts

@@ -0,0 +1,66 @@
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import QueueService from '#services/QueueService'
+
+/** Files per transaction when moving from the table to BullMQ. */
+const FEED_BATCH = 5000
+
+/** Priorities: lower goes first. */
+export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const
+
+/**
+ * The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)).
+ * Producers add file ids to the inspection_queue table; feed() moves them
+ * into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs
+ * wait there, where worker:collection and inspecting nodes take them as
+ * before. Redis holds a working set instead of a backlog of days.
+ */
+class InspectionQueue {
+  readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000)
+
+  /** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */
+  async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) {
+    const ids = [...new Set(fileIds)]
+    for (let i = 0; i < ids.length; i += 10_000) {
+      await db.rawQuery(
+        `INSERT INTO inspection_queue (file_id, priority)
+         SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id
+         ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`,
+        [priority, ids.slice(i, i + 10_000)]
+      )
+    }
+  }
+
+  /** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */
+  async feed(): Promise<number> {
+    const queue = QueueService.collectionQueue
+    const counts = await queue.getJobCounts('wait', 'prioritized')
+    let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0)
+    let moved = 0
+    while (room > 0) {
+      const n = await db.transaction(async (trx) => {
+        const { rows } = await trx.rawQuery(
+          `SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`,
+          [Math.min(room, FEED_BATCH)]
+        )
+        if (rows.length === 0) return 0
+        const ids = rows.map((row: { file_id: string }) => Number(row.file_id))
+        // Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id).
+        await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } })))
+        await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids])
+        return ids.length
+      })
+      if (n === 0) break
+      moved += n
+      room -= n
+    }
+    return moved
+  }
+
+  async size(): Promise<number> {
+    const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`)
+    return Number(rows[0].n)
+  }
+}
+
+export default new InspectionQueue()

app/services/NameService.ts

@@ -112,7 +112,7 @@ export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; r
   const { par2ReleaseName } = await import('#lib/release/index')
   const listings = await db.rawQuery(
     `SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
-     JOIN release_files rf ON rf.release_id = r.id GROUP BY r.id, r.name`
+     JOIN release_files rf ON rf.release_id = r.id WHERE r.source <> 'spot' GROUP BY r.id, r.name`
   )
   let renamed = 0
   for (const release of listings.rows) {

app/services/NntpService.ts

@@ -1,5 +1,5 @@
 import nntpConfig from '#config/nntp'
-import { NNTP } from "nntp-js";
+import { NNTP } from "#lib/nntp/nntp_js";
 import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
 
 class NntpService {
@@ -7,6 +7,8 @@ class NntpService {
   private allConnections: Set<any>;
   private idleConnections: any[];
   private waiters: ((conn: any) => void)[];
+  /** Background work (archive walks): served only when no one else waits. */
+  private lowWaiters: ((conn: any) => void)[] = [];
   private createdCount: number;
 
   constructor(poolSize = 10) {
@@ -34,7 +36,8 @@ class NntpService {
     return conn;
   }
 
-  public async acquire() {
+  /** A connection from the pool; with `low`, after every waiting normal caller. */
+  public async acquire(low = false) {
     if (this.idleConnections.length > 0) {
       return this.idleConnections.pop();
     }
@@ -45,7 +48,7 @@ class NntpService {
       return this._createConnection();
     }
 
-    return new Promise(resolve => this.waiters.push(resolve));
+    return new Promise(resolve => (low ? this.lowWaiters : this.waiters).push(resolve));
   }
 
   /**
@@ -58,7 +61,7 @@ class NntpService {
       this.discard(conn);
       return;
     }
-    const resolve = this.waiters.shift();
+    const resolve = this.nextWaiter();
     if (resolve) {
       resolve(conn);
     } else {
@@ -73,13 +76,17 @@ class NntpService {
     this._destroy(conn);
 
     // Someone is waiting for a slot: open a fresh connection for them.
-    const resolve = this.waiters.shift();
+    const resolve = this.nextWaiter();
     if (resolve) {
       this.createdCount++;
       this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
     }
   }
 
+  private nextWaiter() {
+    return this.waiters.shift() ?? this.lowWaiters.shift();
+  }
+
   private _destroy(conn: any) {
     try {
       conn.sock?.destroy();
@@ -94,6 +101,20 @@ class NntpService {
     return parseGroupResponse(resp)
   }
 
+  /**
+   * Rejects when `promise` hasn't settled within `ms`. nntp-js has no
+   * timeouts: a connection the server stops answering (still ESTABLISHED,
+   * nothing in flight) leaves its command waiting forever. The caller must
+   * then release the connection as failed, so it's destroyed.
+   */
+  public timed<T>(promise: Promise<T>, what: string, ms = nntpConfig.commandTimeoutMs): Promise<T> {
+    let timer: NodeJS.Timeout
+    const timeout = new Promise<never>((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`NNTP ${what}: no answer within ${ms / 1000} s`)), ms)
+    })
+    return Promise.race([promise, timeout]).finally(() => clearTimeout(timer))
+  }
+
   /** Fetches a raw (still dot-stuffed) article body as a Buffer. */
   public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
     const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
@@ -112,6 +133,7 @@ class NntpService {
     this.allConnections.clear();
     this.idleConnections.length = 0;
     this.waiters.length = 0;
+    this.lowWaiters.length = 0;
     this.createdCount = 0;
   }
 }

app/services/NzbImportService.ts

@@ -7,6 +7,9 @@ import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
 import { parseSubject } from '#lib/subject/index'
 import { fileStem } from '#lib/release/index'
 import { looksRandom } from '#services/ArchiveService'
+import FileStore from '#services/FileStore'
+import { isArchiveStart } from '#lib/content/index'
+import InspectionQueue from '#services/InspectionQueue'
 
 export interface NzbImportResult {
   releaseId: number
@@ -20,8 +23,29 @@ export interface NzbImportResult {
 /** Filename of an NZB entry, from its subject (tolerating odd quoting). */
 export function nzbFileName(subject: string): string {
   const parsed = parseSubject(subject)
-  const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
-  return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
+  const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
+  if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
+  return unquotedFileName(subject)
+}
+
+/**
+ * A filename from a subject without quotes, as some spot NZBs have:
+ * `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
+ * `06 - Song.flac  Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
+ * trailing size are dropped; then the name ends at the first extension
+ * followed by a space.
+ */
+export function unquotedFileName(subject: string): string {
+  const cleaned = subject
+    .replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
+    .replace(/\s\d+\s*$/, ' ')
+    .replace(/(\s+-)+\s*$/, '')
+    .replace(/^\s*-\s+/, '')
+    .replace(/\s+/g, ' ')
+    .trim()
+  const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
+  const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
+  return name.replace(/\s+-$/, '').trim() || subject.trim()
 }
 
 export interface NzbImportOptions {
@@ -36,6 +60,13 @@ export interface NzbImportOptions {
   name?: string
   /** Release source for new releases (default `nzb`). */
   source?: ReleaseIdentity['source']
+  /**
+   * The NZB is authoritative for which files form the release and what it is
+   * called (Spotnet spots): its files get the NZB's own release, named
+   * `name`, also when name rules had put them in another release. Files
+   * another trusted NZB claimed first stay where they are.
+   */
+  trusted?: boolean
 }
 
 /**
@@ -55,21 +86,39 @@ class NzbImportService {
       firstId: f.segments[0].id,
     }))
 
-    return db.transaction(async (trx) => {
+    const added: { id: number; filename: string }[] = []
+    const result = await db.transaction(async (trx) => {
       const known = await trx
         .from('files')
-        .whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
-        .select('id', 'release_id', `message_ids`)
-      const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
+        .whereIn('first_message_id', entries.map((e) => e.firstId))
+        .select('id', 'release_id', 'first_message_id')
+      const knownIds = new Set(known.map((f) => f.first_message_id))
 
       // Join the release the listed files are already in, else the posted NZB's own, else a new one.
       const counts = new Map<number, number>()
       for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
       let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
       const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
-      if (!releaseId) {
-        const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
+      const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
       const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
+
+      // Releases of other trusted NZBs: their files are not taken over.
+      let claimed = new Set<number>()
+      if (options.trusted) {
+        // Only while that NZB still stands: a disposed spot's files go to its correction.
+        const holders = await trx
+          .from('releases as r')
+          .whereIn('r.id', [...counts.keys()])
+          .where('r.source', 'spot')
+          .whereNot('r.release_key', key)
+          .whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
+          .select('r.id')
+        claimed = new Set(holders.map((r) => Number(r.id)))
+        // Join another spot's release when it holds most of these files (a repost); else use our own.
+        if (releaseId && !claimed.has(releaseId)) releaseId = undefined
+        if (releaseId) claimed.delete(releaseId)
+      }
+      if (!releaseId) {
         const ids = await IngestService.upsertReleases(trx.knexClient, [
           {
             release: { key, name: title, source: options.source ?? 'nzb' },
@@ -84,9 +133,8 @@ class NzbImportService {
       const now = new Date()
       const missing = entries.filter((e) => !knownIds.has(e.firstId))
       for (let i = 0; i < missing.length; i += 500) {
-        await trx
-          .knexClient('files')
-          .insert(
+        added.push(
+          ...(await FileStore.insert(
             missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
               file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
               filename: name,
@@ -94,22 +142,23 @@ class NzbImportService {
               date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
               parts: file.segments.length,
               size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
-              message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
+              segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
               groups: JSON.stringify(file.groups),
               release_id: releaseId,
               created_at: now,
               updated_at: now,
-            }))
+            })),
+            trx
+          ))
         )
-          .onConflict('file_key')
-          .ignore()
       }
       // The poster's NZB says which files form the post: known files join this
       // release, also when name rules had split them over several.
-      const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
+      const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
+      const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
       await trx
         .from('files')
-        .whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
+        .whereIn('id', moving.map((f) => f.id))
         .update({ release_id: releaseId })
       if (previous.length) {
         await trx.rawQuery(
@@ -130,9 +179,15 @@ class NzbImportService {
 
       const release = await trx.from('releases').where('id', releaseId).first()
       const update: Record<string, unknown> = {}
-      if (nzb.meta.password) update.password = nzb.meta.password
+      if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
       if (!release.file_total) update.file_total = entries.length
-      if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
+      if (options.trusted) {
+        // Only its own release: a repost joining another spot's release keeps that one's title.
+        if (release.release_key === key && release.name !== title) update.name = title
+        if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
+      } else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
+        update.name = title
+      }
       if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
 
       const result = {
@@ -149,6 +204,12 @@ class NzbImportService {
       )
       return result
     })
+    // Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
+    const archives = added.filter((f) => isArchiveStart(f.filename))
+    if (archives.length) {
+      await InspectionQueue.add(archives.map((f) => Number(f.id)))
+    }
+    return result
   }
 
   /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */

app/services/Par2Service.ts

@@ -1,14 +1,10 @@
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
-import NntpService from '#services/NntpService'
 import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
-import { decodeSegment, YencFile } from '#lib/yenc/index'
-import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
+import type { Par2Set } from '#lib/par2/index'
 import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
 import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
 
-/** Par2 index files are small; recovery volumes can be large and are not needed. */
-const MAX_PARTS = 20
 /** Size-only matching below this is too ambiguous. */
 const SIZE_ONLY_MIN = 1024 * 1024
 
@@ -31,36 +27,6 @@ export interface Par2Result {
  * body-derived binaries carry their exact size. See CONCEPT.md (D16).
  */
 class Par2Service {
-  /**
-   * Reads a par2 file whose first segment is already decoded, downloading the
-   * remaining parts if there are any. Returns null when it is not par2.
-   */
-  async read(first: Buffer, messageIds: Record<string, { id: string }>): Promise<Par2Set | null> {
-    if (!isPar2(first)) return null
-    const ids = Object.entries(messageIds)
-      .sort(([a], [b]) => Number(a) - Number(b))
-      .map(([, part]) => part.id)
-    if (ids.length > MAX_PARTS) return null
-    if (ids.length === 1) return parsePar2(first)
-
-    let assembled: YencFile | undefined
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      for (const id of ids) {
-        const segment = decodeSegment(await NntpService.fetchBody(conn, id))
-        assembled ??= YencFile.fromSegment(segment)
-        assembled.add(segment)
-      }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    return parsePar2(assembled!.getBuffer())
-  }
-
   async attach(binary: any, set: Par2Set): Promise<Par2Result> {
     const result = await db.transaction(async (trx) => {
       // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
@@ -143,9 +109,11 @@ class Par2Service {
       )
 
       // The set may have joined a release named after one random filename.
-      let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name
+      const release = await trx.from('releases').where('id', releaseId).first()
+      let releaseName = release?.name
       const better = par2ReleaseName(set.files.map((f) => f.name))
-      if (releaseName && better && isBetterName(releaseName, better)) {
+      // A spot's title is kept (CONCEPT.md D29).
+      if (releaseName && better && release.source !== 'spot' && isBetterName(releaseName, better)) {
         await trx.from('releases').where('id', releaseId).update({ name: better })
         releaseName = better
       }

app/services/QueueService.ts

@@ -41,6 +41,10 @@ class QueueService {
   get collectionQueue() {
     return this.queue('collection-queue')
   }
+  /** Complete archive listings, at low priority (ArchiveWalkService). */
+  get archiveWalkQueue() {
+    return this.queue('archive-walk-queue')
+  }
 
   /** Queues no longer used by the pipeline, kept here so GC can empty them. */
   get legacy(): Queue[] {
@@ -48,7 +52,7 @@ class QueueService {
   }
 
   get all(): Queue[] {
-    return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue]
+    return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue, this.archiveWalkQueue]
   }
 
   /** Closes the queues that were opened. */

app/services/ReleaseCompleteness.ts

@@ -0,0 +1,35 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** Noted releases taken per transaction. */
+const BATCH = 2000
+
+/**
+ * Release completeness (D24) for releases whose files or par2 listing
+ * changed: triggers only note them in release_dirty (a release can be noted
+ * thousands of times between sweeps), and gc:scheduler recomputes each
+ * noted release once every RELEASE_COMPLETE_INTERVAL_MS (LOG.md 2026-10-06 (6)).
+ */
+class ReleaseCompleteness {
+  /** Recomputes all noted releases; returns how many distinct releases were checked. */
+  async sweep(): Promise<number> {
+    let checked = 0
+    for (;;) {
+      const done = await db.transaction(async (trx) => {
+        // Taken and recomputed in one transaction: a failure leaves the notes for the next sweep.
+        const { rows } = await trx.rawQuery(
+          `DELETE FROM release_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM release_dirty LIMIT ?)) RETURNING release_id`,
+          [BATCH]
+        )
+        const ids: number[] = [...new Set<number>(rows.map((row: { release_id: string }) => Number(row.release_id)))]
+        if (ids.length > 0) {
+          await trx.rawQuery(`SELECT recompute_release_complete(id) FROM unnest(?::bigint[]) AS id`, [ids])
+        }
+        checked += ids.length
+        return rows.length < BATCH
+      })
+      if (done) return checked
+    }
+  }
+}
+
+export default new ReleaseCompleteness()

app/services/ReleaseMergeService.ts

@@ -0,0 +1,42 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/**
+ * Uploads split per poster (CONCEPT.md D35). Some uploaders post every
+ * volume of a RAR set under a different poster; the post-id grouping then
+ * makes one release per volume, all with the same name
+ * (`Masters.of.the.Universe.1987…`: 191 one-file releases of one 199-volume
+ * set). Such releases are one upload when all their files share one stem and
+ * no file name repeats (a repost would repeat them, another upload would
+ * bring another stem); their files move into the oldest of them, and GC
+ * deletes the emptied ones.
+ */
+class ReleaseMergeService {
+  async mergeSplitPosts({ dryRun = false } = {}): Promise<{ names: number; releases: number }> {
+    const { rows } = await db.rawQuery(`
+      WITH dup AS (
+        SELECT name FROM releases
+        WHERE file_count > 0 AND source = 'post' AND is_readable_name(name)
+        GROUP BY name HAVING count(*) > 1
+      )
+      SELECT r.name, array_agg(DISTINCT r.id ORDER BY r.id) AS ids
+      FROM dup JOIN releases r ON r.name = dup.name AND r.file_count > 0 AND r.source = 'post'
+      JOIN files f ON f.release_id = r.id
+      GROUP BY r.name
+      HAVING count(DISTINCT name_stem(coalesce(f.real_name, f.filename))) = 1
+        AND count(f.id) = count(DISTINCT coalesce(f.real_name, f.filename))
+        AND max(r.posted_at) - min(r.posted_at) < interval '30 days'`)
+    let releases = 0
+    for (const { name, ids } of rows as { name: string; ids: (string | number)[] }[]) {
+      const [target, ...others] = ids.map(Number)
+      releases += others.length
+      if (dryRun) continue
+      await db.from('files').whereIn('release_id', others).update({ release_id: target })
+      await db.from('binaries').whereIn('release_id', others).update({ release_id: target })
+      logger.info(`Merged ${others.length + 1} releases of one upload split per poster into ${target} "${name}".`)
+    }
+    return { names: rows.length, releases }
+  }
+}
+
+export default new ReleaseMergeService()

app/services/RemoteContentService.ts

@@ -0,0 +1,168 @@
+import { randomUUID } from 'node:crypto'
+import { Job, Worker } from 'bullmq'
+import logger from '@adonisjs/core/services/logger'
+import queueConfig from '#config/queue'
+import nntpConfig from '#config/nntp'
+import QueueService from '#services/QueueService'
+import NntpService from '#services/NntpService'
+import ContentService, { describeContent } from '#services/ContentService'
+import ArchiveWalkService from '#services/ArchiveWalkService'
+import { isMissingArticle, type ContentTask, type InspectProbe, type InspectTask, type WalkProbe } from '#lib/content/probe'
+
+/** Inspection first; archive walks (D40) only when it has nothing waiting. */
+const QUEUES = ['collection-queue', 'archive-walk-queue'] as const
+type QueueName = (typeof QUEUES)[number]
+
+/** A node has this long to post a job's result; after that the job goes back to the queue (BullMQ's stalled check). */
+const LOCK_MS = 10 * 60 * 1000
+
+export interface RemoteJob {
+  queue: QueueName
+  jobId: string
+  /** The job's lock: whoever holds it may complete the job. */
+  token: string
+  task: ContentTask
+}
+
+export class JobLost extends Error {}
+export class ArticleRelayOff extends Error {}
+
+/**
+ * Content inspection and archive walks on seeding nodes (CONCEPT.md D41).
+ * Nodes take jobs from the same queues worker:collection works on: a lease
+ * locks a job (BullMQ's manual processing) and gives the node a task that
+ * was prepared from the database; the node probes it, fetching articles
+ * through the relay (articles()), and posts the result, which is applied
+ * here. A job whose node goes quiet is unlocked after LOCK_MS and picked up
+ * again by whoever comes next.
+ */
+class RemoteContentService {
+  private readonly workers = new Map<QueueName, Worker>()
+
+  /** Up to `max` jobs for a node, each locked for LOCK_MS. */
+  async lease(nodeId: number, max: number): Promise<RemoteJob[]> {
+    const jobs: RemoteJob[] = []
+    for (const queue of QUEUES) {
+      while (jobs.length < max) {
+        const token = `node-${nodeId}:${randomUUID()}`
+        const job = await this.worker(queue).getNextJob(token, { block: false })
+        if (!job) break
+        try {
+          const task = await this.prepare(queue, job)
+          if (!task) {
+            await job.moveToCompleted(null, token, false)
+            continue
+          }
+          // What applying needs, kept with the job: the node's copy of the task isn't trusted.
+          if (task.kind === 'inspect') await job.updateData({ ...job.data, remote: { fileId: task.fileId, name: task.name } })
+          jobs.push({ queue, jobId: job.id!, token, task })
+        } catch (error: any) {
+          await job.moveToFailed(error, token, false).catch(() => {})
+        }
+      }
+    }
+    return jobs
+  }
+
+  /** Applies a node's result and completes the job. */
+  async finish(queue: QueueName, jobId: string, token: string, result: InspectProbe | WalkProbe) {
+    const job = await this.job(queue, jobId)
+    let summary: unknown
+    if (queue === 'collection-queue') {
+      const remote = job.data.remote
+      if (!remote) throw new JobLost(`Job ${jobId} was not leased to a node`)
+      const task: InspectTask = { kind: 'inspect', fileId: Number(remote.fileId), name: String(remote.name), segments: [], sevenZip: null }
+      const content = await ContentService.apply(task, result as InspectProbe)
+      for (const line of describeContent(task.name, content)) logger.info(`[node] ${line}`)
+      summary = content.contentType
+    } else {
+      const outcome = await ArchiveWalkService.apply(Number(job.data.releaseId), result as WalkProbe)
+      logger.info(
+        `[node] Archive walk of release ${outcome.releaseId}: ${outcome.before} → ${outcome.entries} entries from ${outcome.segments} segments` +
+          (outcome.complete ? '' : ` (stopped: ${outcome.stopped})`) +
+          (outcome.renamed ? `, renamed to "${outcome.renamed}"` : '')
+      )
+      summary = outcome
+    }
+    await this.locked(() => job.moveToCompleted(summary, token, false))
+  }
+
+  /** The probe failed on the node (e.g. a missing article): retried like a local failure. */
+  async fail(queue: QueueName, jobId: string, token: string, error: string) {
+    const job = await this.job(queue, jobId)
+    // A missing first article from a node that predates the probe's own handling: a result,
+    // not something to retry (each try costs Eweka ~450 ms; LOG.md 2026-10-06 (14)).
+    if (queue === 'collection-queue' && job.data.remote && isMissingArticle(error)) {
+      return this.finish(queue, jobId, token, { size: 0, contentType: 'missing', md5_16k: null, yencName: '' })
+    }
+    await this.locked(() => job.moveToFailed(new Error(`node: ${error}`), token, false))
+  }
+
+  /** A stopping node gives its jobs back, without using up an attempt. */
+  async release(queue: QueueName, jobId: string, token: string) {
+    const job = await this.job(queue, jobId)
+    await this.locked(() => job.moveToWait(token))
+  }
+
+  /** One article body for a relay node, on this process's NNTP pool. Null when the server doesn't have it. */
+  async article(messageId: string): Promise<Buffer | null> {
+    // NNTP_POOL_SIZE 0 (SEED_ARTICLE_CONNECTIONS=0): no connections for this; acquire() would wait forever.
+    if (nntpConfig.poolSize <= 0) throw new ArticleRelayOff('Article relay is off (SEED_ARTICLE_CONNECTIONS=0)')
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
+    } catch (error: any) {
+      // 430 no such article: a fact about the article, the connection is fine.
+      if (/^4\d\d/.test(String(error.message))) return null
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+
+  /** Closes the lease workers' Redis connections (tests; a server just exits). */
+  async close() {
+    await Promise.all([...this.workers.values()].map((worker) => worker.close()))
+    this.workers.clear()
+  }
+
+  isQueue(name: unknown): name is QueueName {
+    return QUEUES.includes(name as QueueName)
+  }
+
+  private async prepare(queue: QueueName, job: Job): Promise<ContentTask | null> {
+    if (queue === 'collection-queue') return ContentService.prepare(Number(job.data.fileId))
+    const prepared = await ArchiveWalkService.prepare(Number(job.data.releaseId))
+    return 'task' in prepared ? prepared.task : null
+  }
+
+  private async job(queue: QueueName, jobId: string): Promise<Job> {
+    const job = await Job.fromId(queue === 'collection-queue' ? QueueService.collectionQueue : QueueService.archiveWalkQueue, jobId)
+    if (!job) throw new JobLost(`Job ${jobId} is gone`)
+    return job
+  }
+
+  /** BullMQ refuses a job whose lock another holder has now: the node's result came too late. */
+  private async locked(action: () => Promise<unknown>) {
+    try {
+      await action()
+    } catch (error: any) {
+      if (/lock|not in the active state|Missing key/i.test(String(error.message))) throw new JobLost(error.message)
+      throw error
+    }
+  }
+
+  /** A worker that only hands out jobs (no processor, never runs). */
+  private worker(queue: QueueName): Worker {
+    let worker = this.workers.get(queue)
+    if (!worker) {
+      worker = new Worker(queue, null, { connection: queueConfig.connection, lockDuration: LOCK_MS, autorun: false })
+      this.workers.set(queue, worker)
+    }
+    return worker
+  }
+}
+
+export default new RemoteContentService()

app/services/SearchService.ts

@@ -1,6 +1,20 @@
 import db from '@adonisjs/lucid/services/db'
 import { buildNzb } from '#lib/nzb/index'
 import SpotService from '#services/SpotService'
+import { segmentsOf } from '#services/FileStore'
+import { spotCategoryLabels } from '#lib/spotnet/index'
+
+/**
+ * `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
+ * without asking); `missing`: the archive is encrypted and we don't have
+ * it; `none`: no password needed as far as we know.
+ */
+export type PasswordState = 'known' | 'missing' | 'none'
+
+export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
+  if (hasPassword) return 'known'
+  return encrypted ? 'missing' : 'none'
+}
 
 export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
 
@@ -10,7 +24,7 @@ export interface SearchParams {
   readable?: boolean
   /** Only releases that have all their files. */
   complete?: boolean
-  /** Hide password-protected releases. */
+  /** Hide releases that need a password we don't have. */
   hideEncrypted?: boolean
   source?: string
   poster?: string
@@ -31,6 +45,13 @@ export interface SearchParams {
   hideContainers?: boolean
   /** Include incomplete releases that consist of par2 files only (hidden by default). */
   includePar2Only?: boolean
+  /** Size bounds in bytes (releases without a known size never match). */
+  minSize?: number
+  maxSize?: number
+  /** File extensions (without the dot); a release matches if a file or archive entry has one of them (`releases.extensions`). */
+  extensions?: string[]
+  /** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
+  countLimit?: number
 }
 
 const SORTS: Record<SortOrder, string> = {
@@ -49,8 +70,8 @@ const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
 
 /** Incomplete, and every file is a par2 file: nothing to download but repair data. */
 const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
-  SELECT 1 FROM files f WHERE f.release_id = r.id
-    AND f.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
+  SELECT 1 FROM files f LEFT JOIN file_details d ON d.file_id = f.id WHERE f.release_id = r.id
+    AND d.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
 
 /** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
 function pattern(word: string): string {
@@ -58,6 +79,24 @@ function pattern(word: string): string {
   return `%${escaped.replace(/[._\- ]+/g, '_')}%`
 }
 
+/**
+ * As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
+ * spaces as one space. `The.Block.AU` and `the block au` search the same.
+ */
+export function searchNormalize(text: string): string {
+  return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
+}
+
+/** How long the web UI's totals are reused. */
+const STATS_TTL_MS = 60 * 1000
+
+/**
+ * Counting stops here: a broad search needn't count every match. Each
+ * counted row is a random heap read, so the web UI counts less (it only
+ * shows "1,000+"); Newznab clients page by this total and keep the default.
+ */
+const COUNT_LIMIT = 10_000
+
 /**
  * Searching and browsing releases for the web interface (CONCEPT.md D22).
  */
@@ -66,21 +105,33 @@ class SearchService {
     const where: string[] = [
       VISIBLE_SQL,
       // Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
-      `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status = 'removed')`,
+      `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
     ]
     const bindings: unknown[] = []
 
-    const words = (params.q ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 8)
+    // Every word must occur in the release's search text: its name, archive
+    // listing and file names (CONCEPT.md D33), one trigram index for all.
+    const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
     for (const word of words) {
-      const p = pattern(word)
-      // Every word must match the release name, a file name, or the archive contents.
-      // One trigram-indexed lookup per table, not a subquery per release.
-      where.push(`(r.name ILIKE ? OR r.contents::text ILIKE ?
-        OR r.id IN (SELECT f.release_id FROM files f WHERE f.filename ILIKE ? OR f.real_name ILIKE ?))`)
-      bindings.push(p, p, p, p)
+      where.push(`r.search_text LIKE ?`)
+      bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
     }
     if (params.readable) where.push('is_readable_name(r.name)')
-    if (params.hideEncrypted) where.push('r.encrypted IS NOT TRUE')
+    if (params.minSize !== undefined) {
+      where.push('r.size >= ?')
+      bindings.push(params.minSize)
+    }
+    if (params.maxSize !== undefined) {
+      where.push('r.size <= ?')
+      bindings.push(params.maxSize)
+    }
+    // File types from file names and archive listings (migration …130), via their GIN index.
+    const extensions = (params.extensions ?? []).map((e) => e.toLowerCase().replace(/^\./, '')).filter((e) => /^[a-z0-9]{1,8}$/.test(e)).slice(0, 8)
+    if (extensions.length > 0) {
+      where.push('r.extensions && ?::text[]')
+      bindings.push(extensions)
+    }
+    if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
     if (params.hideContainers) where.push('r.container IS NULL')
     // par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
     if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
@@ -121,6 +172,7 @@ class SearchService {
     const offset = params.offset ?? (page - 1) * perPage
     const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
     const whereSql = where.join(' AND ')
+    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
 
     const [{ rows }, { rows: countRows }] = await Promise.all([
       db.rawQuery(
@@ -131,19 +183,22 @@ class SearchService {
                    SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
                    WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
                 jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
-                r.complete, ${PAR2_ONLY_SQL} AS par2_only
+                r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
+                (SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
          FROM releases r WHERE ${whereSql}
          ORDER BY ${order} LIMIT ? OFFSET ?`,
         [...bindings, perPage, offset]
       ),
-      db.rawQuery(`SELECT count(*) AS total FROM releases r WHERE ${whereSql}`, bindings),
+      db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
     ])
 
     return {
       page,
       perPage,
       offset,
-      total: Number(countRows[0].total),
+      total: Math.min(Number(countRows[0].total), countLimit),
+      /** More than `total` match (the count stops at `countLimit`). */
+      totalCapped: Number(countRows[0].total) > countLimit,
       releases: rows.map((r: any) => ({
         id: Number(r.id),
         name: r.name,
@@ -163,6 +218,10 @@ class SearchService {
         category: r.category,
         container: r.container,
         par2Only: r.par2_only,
+        listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
+        hasPassword: r.has_password,
+        passwordState: passwordState(r.encrypted, r.has_password),
+        spotDescription: r.spot_description,
       })),
     }
   }
@@ -193,14 +252,32 @@ class SearchService {
       ),
     ])
 
-    const spot = release.source === 'spot'
-      ? await db.from('spots').where('release_id', id).select('tag', 'poster', 'category', 'website').first()
-      : null
+    const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
+    const spot = spotRow ? await SpotService.withDetails(spotRow) : null
     return {
       id: Number(release.id),
       name: release.name,
       source: release.source,
-      spot: spot ? { tag: spot.tag, poster: spot.poster, category: spot.category, website: spot.website } : null,
+      spot: spot
+        ? {
+            messageId: spot.message_id,
+            title: spot.title,
+            tag: spot.tag,
+            poster: spot.poster,
+            postedAt: spot.posted_at,
+            website: spot.website,
+            description: spot.description,
+            image: spot.image?.segments
+              ? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
+              : spot.image?.url
+                ? { external: spot.image.url }
+                : null,
+            categories: spotCategoryLabels(spot.category, spot.subcats),
+          }
+        : null,
+      password: release.password ?? null,
+      passwordSource: release.password_source ?? null,
+      passwordState: passwordState(release.encrypted, release.password !== null),
       listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
       poster: release.poster,
       groups: release.groups,
@@ -230,6 +307,22 @@ class SearchService {
     }
   }
 
+  /** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
+  async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
+    const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
+    if (!spot) return null
+    const data = await SpotService.image(spot)
+    if (!data) return null
+    const type =
+      data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
+      : data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
+      : data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
+      : data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
+      : data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
+      : null
+    return type ? { data, type } : null
+  }
+
   async nzb(id: number): Promise<{ name: string; xml: string } | null> {
     let release = await db.from('releases').where('id', id).first()
     if (!release) return null
@@ -244,39 +337,83 @@ class SearchService {
       .from('files')
       .where('release_id', id)
       .orderByRaw('coalesce(real_name, filename)')
-      .select('filename', 'real_name', 'poster', 'date', 'groups', 'message_ids')
+      .select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
     const xml = buildNzb(
       files.map((f) => ({
         name: f.real_name ?? f.filename,
         poster: f.poster,
         date: Number(f.date),
         groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
-        segments: f.message_ids,
+        segments: segmentsOf(f),
       })),
       { title: release.name, password: release.password ?? undefined }
     )
     return { name: release.name, xml }
   }
 
+  /**
+   * Totals for the web UI header. They count millions of rows (minutes with
+   * the backfill running), so they are computed at most once per
+   * STATS_TTL_MS and never twice at a time; callers in between share it.
+   */
   async stats() {
+    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
+      this.statsRefresh ??= this.computeStats()
+        .then((value) => {
+          this.statsCache = { at: Date.now(), value }
+        })
+        // A failed background refresh keeps the previous totals; only a caller that waits sees the error.
+        .catch((error) => {
+          if (!this.statsCache) throw error
+        })
+        .finally(() => {
+          this.statsRefresh = undefined
+        })
+      // Only the very first caller waits; later ones get the previous totals meanwhile.
+      if (!this.statsCache) await this.statsRefresh
+    }
+    return this.statsCache!.value
+  }
+
+  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
+  private statsRefresh?: Promise<void>
+
+  /**
+   * Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
+   * from Postgres's statistics (visible releases = the entries of an index
+   * that covers exactly them), ratios and sizes from a 0.5% sample of the
+   * releases table. Within a few percent once autovacuum has analysed.
+   */
+  private async computeStats() {
     const { rows } = await db.rawQuery(`
-      SELECT (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL}) AS releases,
-             (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL} AND is_readable_name(name)) AS readable_releases,
-             (SELECT count(*) FROM files) AS files,
-             (SELECT coalesce(sum(size), 0) FROM files) AS size,
-             (SELECT count(*) FROM releases WHERE encrypted) AS encrypted_releases,
+      WITH sample AS (
+        SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
+        FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
+      SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
+             -- files is partitioned (D39): its partitions hold the counts.
+             (SELECT sum(c.reltuples) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass) AS files,
+             (SELECT count(*) FROM sample) AS sampled,
+             (SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
+             -- Indexed data only: a spot listing's size is what the spot announces.
+             (SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
+             (SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
              (SELECT max(updated_at) FROM groups) AS last_indexed_at,
-             (SELECT json_agg(json_build_object('source', source, 'releases', n) ORDER BY n DESC)
-                FROM (SELECT coalesce(source, 'unknown') AS source, count(*) AS n FROM releases r WHERE ${VISIBLE_SQL} GROUP BY 1) s) AS sources`)
+             (SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
+                FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
     const r = rows[0]
+    const releases = Math.max(0, Number(r.releases))
+    const sampled = Number(r.sampled)
+    // Sample counts scaled to all visible releases.
+    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
     return {
-      releases: Number(r.releases),
-      readableReleases: Number(r.readable_releases),
-      files: Number(r.files),
-      size: Number(r.size),
-      encryptedReleases: Number(r.encrypted_releases),
+      approximate: true,
+      releases,
+      readableReleases: scale(r.sampled_readable),
+      files: Math.max(0, Number(r.files)),
+      size: scale(r.sampled_size),
+      encryptedReleases: scale(r.sampled_encrypted),
       lastIndexedAt: r.last_indexed_at,
-      sources: r.sources ?? [],
+      sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
     }
   }
 }

app/services/SeedFetchPool.ts

@@ -0,0 +1,132 @@
+import nntpConfig from '#config/nntp'
+import indexerConfig from '#config/indexer'
+import { XoverClient } from '#lib/nntp/xover_client'
+
+interface Slot {
+  client: XoverClient
+  group?: string
+  usedAt: number
+}
+
+/**
+ * A pooled connection unused this long is replaced: the news server drops
+ * idle connections without telling, and a request on one then hangs until
+ * the inactivity timeout (LOG.md 2026-10-05 (11)).
+ */
+const IDLE_MS = 60 * 1000
+
+/**
+ * NNTP connections in the seed API for relay nodes (CONCEPT.md D38): the
+ * indexer fetches overview chunks and passes the gzip stream on unopened, so
+ * nodes need no news server account and every connection comes from this
+ * machine's IP. SEED_NNTP_CONNECTIONS (0: relay off).
+ */
+class SeedFetchPool {
+  private idle: Slot[] = []
+  private open = 0
+  private waiters: (() => void)[] = []
+
+  get size() {
+    return indexerConfig.seed.nntpConnections
+  }
+
+  /**
+   * Pooled connections fetch compressed unless SEED_NNTP_COMPRESS=false. A failure is retried once without compression, on a connection of its
+   * own: either the pooled connection was dead (dropped while idle), or the
+   * server stalled mid-reply in compressed mode. Eweka does that for some
+   * chunks: it stops after most of the stream and never sends the rest
+   * (a.b.ath 49701720000: 19,672 of 20,000 overviews, then silence), while
+   * the same chunk uncompressed comes in 0.3 s (LOG.md 2026-10-05 (11)).
+   */
+  async fetch(group: string, from: number, to: number): Promise<Buffer> {
+    try {
+      return await this.fetchOnce(group, from, to)
+    } catch {
+      return this.fetchPlain(group, from, to)
+    }
+  }
+
+  private async fetchPlain(group: string, from: number, to: number): Promise<Buffer> {
+    const client = await this.connect(false)
+    try {
+      await client.group(group)
+      return await client.xoverRaw(from, to)
+    } finally {
+      client.close()
+    }
+  }
+
+  private connect(compress: boolean) {
+    return XoverClient.connect({
+      host: nntpConfig.host,
+      port: nntpConfig.port,
+      secure: nntpConfig.secure,
+      servername: nntpConfig.tlsServername,
+      user: nntpConfig.user,
+      password: nntpConfig.password?.release(),
+      timeoutMs: indexerConfig.seed.nntpTimeoutMs,
+      compress,
+    })
+  }
+
+  private async fetchOnce(group: string, from: number, to: number): Promise<Buffer> {
+    const slot = await this.acquire()
+    let ok = false
+    try {
+      if (slot.group !== group) {
+        await slot.client.group(group)
+        slot.group = group
+      }
+      const raw = await slot.client.xoverRaw(from, to)
+      ok = true
+      return raw
+    } finally {
+      this.release(slot, ok)
+    }
+  }
+
+  /** Closes the idle connections (tests, shutdown); busy ones close when released. */
+  closeIdle() {
+    for (const slot of this.idle.splice(0)) {
+      slot.client.close()
+      this.open--
+    }
+  }
+
+  private async acquire(): Promise<Slot> {
+    while (true) {
+      const slot = this.idle.pop()
+      if (slot) {
+        if (Date.now() - slot.usedAt < IDLE_MS) return slot
+        slot.client.close()
+        this.open--
+        continue
+      }
+      if (this.open < this.size) {
+        this.open++
+        try {
+          return { client: await this.connect(indexerConfig.seed.nntpCompress), usedAt: Date.now() }
+        } catch (error) {
+          this.open--
+          this.waiters.shift()?.()
+          throw error
+        }
+      }
+      await new Promise<void>((resolve) => this.waiters.push(resolve))
+    }
+  }
+
+  private release(slot: Slot, ok: boolean) {
+    if (ok) {
+      slot.usedAt = Date.now()
+      this.idle.push(slot)
+    } else {
+      // A failed command can leave the connection mid-reply: start afresh.
+      slot.client.close()
+      this.open--
+    }
+    this.waiters.shift()?.()
+  }
+}
+
+export default new SeedFetchPool()

app/services/SeedOrchestrator.ts

@@ -0,0 +1,234 @@
+import { createHash, randomBytes } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import { cut, uncovered } from '#lib/seed/plan'
+import NntpService from '#services/NntpService'
+import SeedService from '#services/SeedService'
+import BackfillService from '#services/BackfillService'
+import SeedFetchPool from '#services/SeedFetchPool'
+import { articleAtDate } from '#lib/nntp/date_search'
+import { normalizeSeedFile, type SeedBatch } from '#lib/seed/wire'
+
+/** A lease without a batch for this long is given to another node; the range resumes at next_id. */
+export const LEASE_TTL_SECONDS = 15 * 60
+/** Leases per range before it counts as failed (a range that keeps breaking nodes). */
+const MAX_ATTEMPTS = 5
+/** Articles per XOVER probe when looking for an article by date. */
+const PROBE = 50
+
+export interface SeedLease {
+  id: number
+  group: string
+  /** Resume point: first article not reported yet. */
+  start: number
+  end: number
+}
+
+export class LeaseLost extends Error {}
+
+const hash = (token: string) => createHash('sha256').update(token).digest('hex')
+
+/**
+ * The seed orchestrator (CONCEPT.md D38): seeding nodes lease article
+ * ranges over HTTPS, report what they aggregated in batches (D37), and
+ * finish or give back the range. Nodes only make outgoing requests; a lease
+ * that goes quiet for LEASE_TTL_SECONDS passes to the next node that asks.
+ */
+class SeedOrchestrator {
+  /** Creates a node, or gives an existing one a new token. Returns the token (stored hashed only). */
+  async issueToken(name: string): Promise<string> {
+    const token = `sn_${randomBytes(24).toString('base64url')}`
+    await db
+      .table('seed_nodes')
+      .insert({ name, token_hash: hash(token), disabled: false })
+      .onConflict('name')
+      .merge({ token_hash: hash(token), disabled: false })
+    return token
+  }
+
+  async nodeForToken(token: string): Promise<{ id: number; name: string } | null> {
+    const node = await db.from('seed_nodes').where('token_hash', hash(token)).where('disabled', false).first()
+    if (!node) return null
+    if (!node.last_seen_at || Date.now() - new Date(node.last_seen_at).getTime() > 60_000) {
+      await db.from('seed_nodes').where('id', node.id).update({ last_seen_at: new Date() })
+    }
+    return { id: Number(node.id), name: node.name }
+  }
+
+  /**
+   * The next range for this node, oldest plan first; a quiet lease is taken
+   * over. Nothing while the database is over the backfill's disk limits: the
+   * node asks again a minute later.
+   */
+  async lease(nodeId: number): Promise<SeedLease | null> {
+    if ((await BackfillService.room()) !== true) return null
+    // Fully reported but never finished (the node went away at the very end): nothing left to fetch.
+    await db.rawQuery(
+      `UPDATE seed_ranges SET status = 'done', done_at = now(), node_id = NULL
+       WHERE next_id > end_id AND (status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))`,
+      [LEASE_TTL_SECONDS]
+    )
+    // Ranges that broke node after node stop being handed out.
+    await db.rawQuery(
+      `UPDATE seed_ranges SET status = 'failed', error = coalesce(error, 'lease expired too often')
+       WHERE status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?) AND attempts >= ?`,
+      [LEASE_TTL_SECONDS, MAX_ATTEMPTS]
+    )
+    // Priorities take turns: while ranges of a higher priority are leased for less than half of
+    // the work in progress they go first, otherwise the rest does (a 2026 gap fetches ~5x slower
+    // than older history; LOG.md 2026-10-06 (10)).
+    const { rows: mix } = await db.rawQuery(
+      `SELECT count(*) FILTER (WHERE priority > 0) AS high, count(*) AS all FROM seed_ranges
+       WHERE status = 'leased' AND heartbeat_at >= now() - make_interval(secs => ?)`,
+      [LEASE_TTL_SECONDS]
+    )
+    const order = Number(mix[0].high) * 2 < Number(mix[0].all) + 1 ? 'priority DESC, id' : 'priority ASC, id'
+    const { rows } = await db.rawQuery(
+      `UPDATE seed_ranges SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1
+       WHERE id = (
+         SELECT id FROM seed_ranges
+         WHERE status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?))
+         ORDER BY ${order} LIMIT 1 FOR UPDATE SKIP LOCKED)
+       RETURNING id, group_name, next_id, end_id`,
+      [nodeId, LEASE_TTL_SECONDS]
+    )
+    const row = rows[0]
+    return row ? { id: Number(row.id), group: row.group_name, start: Number(row.next_id), end: Number(row.end_id) } : null
+  }
+
+  /** Stores a node's batch and moves the range's resume point; LeaseLost when the range is no longer this node's. */
+  async batch(nodeId: number, rangeId: number, batch: SeedBatch) {
+    const range = await this.leased(nodeId, rangeId)
+    const stored = await SeedService.store(batch.files.map(normalizeSeedFile))
+    await SeedService.storeFragments(batch.fragments, { start: Number(range.start_id), end: Number(range.end_id) })
+    await db.rawQuery(
+      `UPDATE seed_ranges SET next_id = greatest(next_id, least(?, end_id + 1)), heartbeat_at = now(),
+         headers = headers + ?, files = files + ?, fragments = fragments + ?
+       WHERE id = ?`,
+      [batch.next, batch.headers, batch.files.length, batch.fragments.length, rangeId]
+    )
+    return stored
+  }
+
+  /**
+   * Relay: overviews of `from`..`to` of a leased range, fetched here and handed
+   * on as the gzip stream the news server sent. Keeps the lease alive.
+   */
+  async chunk(nodeId: number, rangeId: number, from: number, to: number): Promise<Buffer> {
+    const range = await this.leased(nodeId, rangeId)
+    const start = Math.max(from, Number(range.start_id))
+    const end = Math.min(to, Number(range.end_id), start + 49_999)
+    if (end < start) throw new RangeError(`Articles ${from}–${to} are outside range ${rangeId}`)
+    await db.from('seed_ranges').where('id', rangeId).update({ heartbeat_at: new Date() })
+    return SeedFetchPool.fetch(range.group_name, start, end)
+  }
+
+  async finish(nodeId: number, rangeId: number) {
+    await this.leased(nodeId, rangeId)
+    await db.from('seed_ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date(), error: null })
+  }
+
+  /**
+   * A node gives a range back (it failed, or is shutting down); it's queued
+   * again unless it failed too often. `uncounted`: the node never started on
+   * it (couldn't reach its news server), so the lease doesn't count as an attempt.
+   */
+  async release(nodeId: number, rangeId: number, error?: string, uncounted = false) {
+    const range = await this.leased(nodeId, rangeId)
+    const attempts = uncounted ? Math.max(0, range.attempts - 1) : range.attempts
+    await db
+      .from('seed_ranges')
+      .where('id', rangeId)
+      .update({ status: attempts >= MAX_ATTEMPTS ? 'failed' : 'queued', node_id: null, attempts, error: error?.slice(0, 2000) ?? null })
+  }
+
+  private async leased(nodeId: number, rangeId: number) {
+    const range = await db.from('seed_ranges').where('id', rangeId).first()
+    if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
+    return range
+  }
+
+  /**
+   * Plans ranges for groups: from below what live indexing and the backfill
+   * cover (or below earlier seed ranges) down to `days` back, or to the
+   * server's oldest article. A group's backfill stops where its seed plan
+   * begins. Ranges are interleaved across groups, newest first.
+   */
+  async plan(
+    groups: string[],
+    options: { days?: number; size: number; priority?: number }
+  ): Promise<{ group: string; from: number; to: number; ranges: number; gaps: number }[]> {
+    const perGroup: { group: string; ranges: [number, number][] }[] = []
+    const summary: { group: string; from: number; to: number; ranges: number; gaps: number }[] = []
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      for (const name of groups) {
+        const group = await db.from('groups').where('name', name).first()
+        if (!group) throw new Error(`Unknown group ${name}`)
+        const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, name), `GROUP ${name}`)
+        const dateOf = async (n: bigint) => {
+          const { overviews } = await NntpService.timed<any>(conn.xover(n, n + BigInt(PROBE)), `XOVER ${name} ${n}`)
+          const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
+          const date = header ? Date.parse(header.date) : Number.NaN
+          return Number.isNaN(date) ? null : date
+        }
+        // Top: where live indexing began, or as far down as the backfill has come.
+        let top = Number(await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf))
+        if (group.backfill_next_id !== null) top = Math.min(top, Number(group.backfill_next_id))
+        const backfilling = group.backfill_next_id !== null && group.backfill_until_id !== null && Number(group.backfill_next_id) > Number(group.backfill_until_id)
+        const bottom = options.days
+          ? Number(await articleAtDate(first, last, Date.now() - options.days * 24 * 60 * 60 * 1000, dateOf))
+          : Number(first)
+
+        // Everything below the top that no earlier range covers (in any state). Planning below
+        // the lowest earlier range left a 13-month gap in a.b.ath (LOG.md 2026-10-06 (7)).
+        const planned = await db.from('seed_ranges').where('group_name', name).select('start_id', 'end_id')
+        const gaps = uncovered(bottom, top - 1, planned.map((r) => [Number(r.start_id), Number(r.end_id)] as [number, number]))
+        const ranges = cut(gaps, options.size)
+        perGroup.push({ group: name, ranges })
+        summary.push({ group: name, from: bottom, to: top - 1, ranges: ranges.length, gaps: gaps.length })
+        // The seed plan takes over the rest of this group's backfill.
+        if (backfilling && ranges.length > 0) {
+          await db.from('groups').where('id', group.id).update({ backfill_until_id: group.backfill_next_id })
+        }
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+
+    // Round-robin over groups, each newest first, so every group fills in from the present backwards.
+    const rows: Record<string, unknown>[] = []
+    for (let i = 0; perGroup.some((g) => i < g.ranges.length); i++) {
+      for (const g of perGroup) {
+        const range = g.ranges[i]
+        if (range) rows.push({ group_name: g.group, start_id: range[0], end_id: range[1], next_id: range[0], priority: options.priority ?? 0 })
+      }
+    }
+    for (let i = 0; i < rows.length; i += 5000) await db.table('seed_ranges').multiInsert(rows.slice(i, i + 5000))
+    return summary
+  }
+
+  async status() {
+    const { rows: ranges } = await db.rawQuery(
+      `SELECT status, count(*)::int AS ranges, sum(end_id - start_id + 1)::bigint AS articles,
+              sum(next_id - start_id)::bigint AS reached, sum(headers)::bigint AS headers, sum(files)::bigint AS files
+       FROM seed_ranges GROUP BY status ORDER BY status`
+    )
+    const { rows: nodes } = await db.rawQuery(
+      `SELECT n.name, n.last_seen_at, n.disabled,
+              (SELECT count(*)::int FROM seed_ranges r WHERE r.node_id = n.id AND r.status = 'leased') AS leases,
+              (SELECT coalesce(sum(headers), 0)::bigint FROM seed_ranges r WHERE r.node_id = n.id) AS headers
+       FROM seed_nodes n ORDER BY n.name`
+    )
+    const { rows: recent } = await db.rawQuery(
+      `SELECT coalesce(sum(headers), 0)::bigint AS headers FROM seed_ranges WHERE done_at > now() - interval '1 hour'`
+    )
+    const room = await BackfillService.room()
+    return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers), paused: room === true ? null : room }
+  }
+}
+
+export default new SeedOrchestrator()

app/services/SeedService.ts

@@ -0,0 +1,152 @@
+import db from '@adonisjs/lucid/services/db'
+import IngestService from '#services/IngestService'
+import FileStore, { insertLimits } from '#services/FileStore'
+import SfvService from '#services/SfvService'
+import ContentService from '#services/ContentService'
+import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
+import InspectionQueue from '#services/InspectionQueue'
+import GarbageFilter from '#services/GarbageFilter'
+
+/** Articles from a range's edge within which a file may continue in the neighbouring range. */
+const EDGE_ARTICLES = 50_000
+
+export interface SeedStoreResult {
+  files: number
+  /** Files that existed already (same file_key, e.g. from live indexing or a crosspost). */
+  existing: number
+  releases: number
+  inspections: number
+  /** Random-named files of garbage signatures or dropped releases, not stored (D42). */
+  dropped: number
+}
+
+/**
+ * Stores what seeding nodes found (CONCEPT.md D37): per complete file its
+ * release, the file with its segment list, and one completed binary, as
+ * worker:file would have left it, so par2 naming, inspection and GC treat
+ * it like any other file; fragments take the regular ingest path. A batch
+ * is a few statements, not one per article.
+ */
+class SeedService {
+  async store(batch: SeedFile[]): Promise<SeedStoreResult> {
+    const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0, dropped: 0 }
+    // A repost can complete the same file twice in one batch; ON CONFLICT DO UPDATE takes each row once.
+    const unique = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
+    // Garbage isn't stored at all: no release, file or binary (CONCEPT.md D42).
+    const { kept: files, dropped } = await GarbageFilter.keep(unique, (file) => ({
+      name: file.name,
+      messageId: file.segments[0]?.[1],
+      groups: file.groups,
+      releaseKey: file.release?.key ?? file.fileKey,
+    }))
+    result.dropped = dropped
+    if (files.length === 0) return result
+    const knex = db.connection().getWriteClient()
+    const now = new Date()
+    const postedOf = (file: SeedFile) => (file.postedAt ? new Date(file.postedAt) : undefined)
+
+    const releaseIds = await IngestService.upsertReleases(
+      knex,
+      files.map((file) => ({ release: file.release, poster: file.poster, groups: file.groups, postedAt: postedOf(file) })),
+      now
+    )
+    result.releases = releaseIds.size
+
+    const rows = files.map((file) => ({
+      file_key: file.fileKey,
+      filename: file.name,
+      poster: file.poster,
+      date: (postedOf(file) ?? now).getTime(),
+      parts: file.segments.length,
+      segments: file.segments.map(([number, id, size]) => ({ number, id, size })),
+      groups: JSON.stringify(file.groups),
+      release_id: file.release ? (releaseIds.get(file.release.key) ?? null) : null,
+      size: file.segments.reduce((sum, [, , size]) => sum + size, 0),
+      created_at: now,
+      updated_at: now,
+    }))
+    const inserted = await FileStore.insert(rows)
+    result.files = inserted.length
+    result.existing = rows.length - inserted.length
+
+    // File ids for the binaries below (files queued for inspection only): new ones from the insert,
+    // existing ones looked up. Looking up every key of the batch probed all files partitions again,
+    // 15% of Postgres' active time (LOG.md 2026-10-06 (12)).
+    const fileIds = new Map<string, number>()
+    for (const row of inserted) if (row.file_key) fileIds.set(row.file_key, row.id)
+    const missing = files.filter((file) => !fileIds.has(file.fileKey) && ContentService.isCandidate(file.name)).map((file) => file.fileKey)
+    for (let i = 0; i < missing.length; i += 5000) {
+      for (const row of await db.from('files').whereIn('file_key', missing.slice(i, i + 5000)).select('id', 'file_key')) {
+        fileIds.set(row.file_key, Number(row.id))
+      }
+    }
+    // A completed binary, as worker:file would have left behind, only for files queued for content
+    // inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others
+    // (~85% in a.b.ath) it was only GC churn: 1.6M rows an hour inserted and deleted (LOG.md 2026-10-06 (11)).
+    const insertedKeys = new Set(inserted.map((row) => row.file_key))
+    const binaries = files.flatMap((file, i) => {
+      if (!ContentService.isCandidate(file.name)) return []
+      return [{
+        file_key: file.fileKey,
+        poster: file.poster,
+        subject: file.subject,
+        name: file.name,
+        total_parts: file.totalParts,
+        groups: rows[i].groups,
+        posted_at: postedOf(file) ?? null,
+        first_seen_at: now,
+        last_seen_at: now,
+        completed_at: now,
+        release_id: rows[i].release_id,
+        file_id: fileIds.get(file.fileKey) ?? null,
+        // New files are queued for inspection below; an existing file was queued when it was added.
+        awaiting_inspection: insertedKeys.has(file.fileKey) ? true : null,
+      }]
+    })
+    binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
+    for (let i = 0; i < binaries.length; i += insertLimits.rows) {
+      await knex('binaries')
+        .insert(binaries.slice(i, i + insertLimits.rows))
+        .onConflict('file_key')
+        .merge({
+          file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any,
+          awaiting_inspection: knex.raw('coalesce(binaries.awaiting_inspection, excluded.awaiting_inspection)') as any,
+        })
+    }
+
+    // As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
+    const rowByKey = new Map(rows.map((row) => [row.file_key, row]))
+    const added = inserted.flatMap((file) => {
+      const row = file.file_key ? rowByKey.get(file.file_key) : undefined
+      return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
+    })
+    await SfvService.claimAll(added)
+    const candidates = added
+      .filter((file) => ContentService.isCandidate(file.name))
+      .map((file) => file.id)
+    if (candidates.length > 0) await InspectionQueue.add(candidates)
+    result.inspections = candidates.length
+    return result
+  }
+
+  /**
+   * Segments of files that didn't complete on a node. Only files near the
+   * range's edges can continue in the neighbouring range (worked on by some
+   * node, maybe hours later): those go straight into binaries and parts
+   * (staging would drop them after 30 minutes). The rest are files with
+   * articles missing for good, and are dropped: staging them filled 22 GB
+   * of churn on the NVMe (LOG.md 2026-10-05 (10)). Returns the segments kept.
+   */
+  async storeFragments(fragments: SeedFragment[], range: { start: number; end: number }): Promise<number> {
+    const nearEdge = new Set<string>()
+    for (const fragment of fragments) {
+      const n = Number(fragment.articleNumber)
+      if (Number.isFinite(n) && (n < range.start + EDGE_ARTICLES || n > range.end - EDGE_ARTICLES)) nearEdge.add(fragment.fileKey)
+    }
+    const kept = fragments.filter((fragment) => nearEdge.has(fragment.fileKey))
+    if (kept.length > 0) await IngestService.ingest(kept.map(fromFragment))
+    return kept.length
+  }
+}
+
+export default new SeedService()

app/services/SevenZipService.ts

@@ -0,0 +1,36 @@
+import db from '@adonisjs/lucid/services/db'
+import { sevenZipVolume } from '#lib/sevenzip/index'
+
+/**
+ * The database side of 7z listings (CONCEPT.md D19): which volumes make up
+ * an archive. Listing itself (first segment, tail, bsdtar) is
+ * lib/sevenzip/list.ts, run by inspection here or on a node.
+ */
+class SevenZipService {
+  /**
+   * The volumes of the archive `file` belongs to, in order, when all of them
+   * up to the highest number seen are there; null otherwise. An unsplit (or
+   * unnamed) archive is its own only volume.
+   */
+  async volumes(file: any): Promise<any[] | null> {
+    const name = String(file.real_name ?? file.filename)
+    const volume = sevenZipVolume(name)
+    if (!volume) return [file]
+    if (!file.release_id) return null
+    const base = name.replace(/\.\d+$/, '').toLowerCase()
+    const rows = await db
+      .from('files')
+      .where('release_id', file.release_id)
+      .whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`])
+    const byNumber = new Map<number, any>()
+    for (const row of rows) {
+      const n = sevenZipVolume(String(row.real_name ?? row.filename))
+      if (n && !byNumber.has(n)) byNumber.set(n, row)
+    }
+    const max = Math.max(0, ...byNumber.keys())
+    const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
+    return max > 0 && ordered.every(Boolean) ? ordered : null
+  }
+}
+
+export default new SevenZipService()

app/services/SfvService.ts

@@ -0,0 +1,96 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import FileStore from '#services/FileStore'
+import { parseSfv, sfvReleaseName } from '#lib/content/index'
+
+/** Releases made by name rules alone; an SFV's listing outranks them. */
+const WEAK_SOURCES = ['stem', 'weak-stem', 'subject']
+/** Files further apart than this from the SFV in posting time aren't taken. */
+const WINDOW_MS = 2 * 24 * 60 * 60 * 1000
+
+/**
+ * SFV grouping (CONCEPT.md D34). An uploader that posts every track of an
+ * album as its own file (13.5k such releases from one poster on homelab)
+ * still posts the album's SFV, which names them all.
+ */
+class SfvService {
+  /**
+   * Records an SFV file's listing on its release, moves the listed files that
+   * sit in weak releases into it, and names the release after the album when
+   * it's still named after a track or the SFV itself. Returns how many files
+   * moved, or null when it isn't an SFV.
+   */
+  async attach(sfvFile: any, text: string): Promise<number | null> {
+    const entries = parseSfv(text)
+    if (entries.length === 0 || !sfvFile.release_id) return null
+    const releaseId = Number(sfvFile.release_id)
+    const postedAt = sfvFile.date ? new Date(Number(sfvFile.date)) : null
+    await db
+      .knexQuery()
+      .table('sfv_entries')
+      .insert(entries.map((e) => ({ release_id: releaseId, name: e.name, crc: e.crc, poster: sfvFile.poster, posted_at: postedAt })))
+      .onConflict(['release_id', 'name'])
+      .ignore()
+    await FileStore.setDetails(Number(sfvFile.id), { content_type: 'sfv' })
+
+    const candidates = await db
+      .from('files as f')
+      .join('releases as r', 'r.id', 'f.release_id')
+      .whereIn(db.raw('coalesce(f.real_name, f.filename)') as any, entries.map((e) => e.name))
+      .whereNot('f.release_id', releaseId)
+      .whereIn('r.source', WEAK_SOURCES)
+      .select('f.id', 'f.poster', 'f.date')
+    const moving = candidates.filter((f) => this.sameUpload(f, sfvFile.poster, postedAt))
+    await this.move(moving.map((f) => Number(f.id)), releaseId)
+
+    const release = await db.from('releases').where('id', releaseId).first()
+    const album = sfvReleaseName(String(sfvFile.real_name ?? sfvFile.filename))
+    if (release && WEAK_SOURCES.includes(release.source) && album !== release.name && /^\d{1,3}-/.test(release.name)) {
+      await db.from('releases').where('id', releaseId).update({ name: album })
+    }
+    logger.info(`SFV "${sfvFile.filename}": ${entries.length} files listed, ${moving.length} moved into release ${releaseId} "${album}".`)
+    return moving.length
+  }
+
+  /** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
+  /**
+   * claim() for a batch of new files: one lookup for all their names, the
+   * per-file work only for the few an SFV lists (seeding stores thousands
+   * of files per batch; CONCEPT.md D37).
+   */
+  async claimAll(files: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }[]): Promise<number> {
+    if (files.length === 0) return 0
+    const listed = new Set(
+      (await db.from('sfv_entries').whereIn('name', [...new Set(files.map((f) => f.name))]).select('name')).map((e) => e.name as string)
+    )
+    let moved = 0
+    for (const file of files) if (listed.has(file.name) && (await this.claim(file)) !== null) moved++
+    return moved
+  }
+
+  async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
+    const entries = await db.from('sfv_entries').where('name', file.name)
+    const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))
+    if (!entry || Number(entry.release_id) === Number(file.release_id)) return null
+    if (file.release_id) {
+      const current = await db.from('releases').where('id', file.release_id).first()
+      if (current && !WEAK_SOURCES.includes(current.source)) return null
+    }
+    await this.move([file.id], Number(entry.release_id))
+    return Number(entry.release_id)
+  }
+
+  private sameUpload(file: { poster?: string | null; date?: number | string | null }, poster: string | null, postedAt: Date | null) {
+    if (poster && file.poster && file.poster !== poster) return false
+    if (postedAt && file.date && Math.abs(Number(file.date) - postedAt.getTime()) > WINDOW_MS) return false
+    return true
+  }
+
+  private async move(fileIds: number[], releaseId: number) {
+    if (fileIds.length === 0) return
+    await db.from('files').whereIn('id', fileIds).update({ release_id: releaseId })
+    await db.from('binaries').whereIn('file_id', fileIds).update({ release_id: releaseId })
+  }
+}
+
+export default new SfvService()

app/services/SpotService.ts

@@ -1,28 +1,58 @@
+import { createHash } from 'node:crypto'
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import indexerConfig from '#config/indexer'
 import NntpService from '#services/NntpService'
 import NzbImportService from '#services/NzbImportService'
-import { type OverviewHeader } from '#services/IngestService'
+import IngestService, { type OverviewHeader } from '#services/IngestService'
 import { parseNzb } from '#lib/nzb/index'
 import {
   moderationTarget,
+  passwordFromSpot,
   spotterId,
   decodeSpotBinary,
+  decodeSpotText,
   parseSpotArticleHeaders,
   parseSpotHeader,
   parseSpotXml,
   verifySpotHeader,
   type SpotHeader,
+  type SpotXml,
 } from '#lib/spotnet/index'
 
 /** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
 const MAX_NZB_SEGMENTS = 20
 /** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
 const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
+/** Rows per INSERT: 14 columns each, under Postgres' 65,535 bind parameters. */
+const INSERT_CHUNK = 2000
+/** Spot images larger than this many articles aren't fetched. */
+const MAX_IMAGE_SEGMENTS = 10
 /** A claimed spot whose worker died is taken again after this long. */
 const RECLAIM_AFTER = '1 hour'
 
+/** The NZB's articles were read, but they don't decode to an NZB (e.g. "invalid code lengths set"). */
+export class NzbUnreadable extends Error {}
+
+/**
+ * A spot's title: from its XML, which is decoded properly (the Subject of a
+ * raw header is read as ISO-8859-1), else from the header.
+ */
+function spotTitle(head: { header: SpotHeader; xml: SpotXml | null }): string {
+  return head.xml?.title || head.header.title
+}
+
+/**
+ * A placeholder NZB (e.g. Dummy.rar + Dummy.nzb, 2.6 KB, shared by many
+ * spots): less than 1% of the size the spot announces (the real NZB is
+ * handed out elsewhere), or under 16 KB in all (text-only spots: requests,
+ * announcements). Its files are not imported, and the spot is hidden.
+ */
+export function isDummyNzb(nzb: { files: { segments: { bytes: number }[] }[] }, spotSize: number): boolean {
+  const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
+  return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
+}
+
 export function isSpotGroup(name: string): boolean {
   return indexerConfig.spotGroups.includes(name)
 }
@@ -55,74 +85,115 @@ class SpotService {
         posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
         key_id: spot.keyId,
         spotter_id: spotterId(spot.selfSignedPubKey) || null,
+        // From the overview; titles with accents may fail here (UTF-8 decoding)
+        // and are verified again on their raw header by worker:spots.
+        verified: verifySpotHeader(spot),
         created_at: now,
       })
     }
-    if (rows.length === 0) return 0
-    await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
+    for (let i = 0; i < rows.length; i += INSERT_CHUNK) {
+      await db.knexQuery().table('spots').insert(rows.slice(i, i + INSERT_CHUNK)).onConflict('message_id').ignore()
+    }
     return rows.length
   }
 
-  /** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
+  /**
+   * Takes up to `limit` pending spots, marked `processing`: recent ones
+   * (to be imported) first, then the history in posting order, so live spots
+   * don't wait behind a backfill and moderation follows its targets.
+   */
   async claim(limit: number): Promise<any[]> {
-    const { rows } = await db.rawQuery(
+    const claim = async (filter: string, bindings: unknown[], n: number) =>
+      (
+        await db.rawQuery(
           `UPDATE spots SET status = 'processing', processed_at = now()
            WHERE message_id IN (
-         SELECT message_id FROM spots
-         WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
-         ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
+             SELECT message_id FROM spots WHERE ${filter}
+             ORDER BY article_number LIMIT ? FOR UPDATE SKIP LOCKED)
            RETURNING *`,
-      [limit]
+          [...bindings, n]
         )
+      ).rows as any[]
+    const rows = await claim(`status = 'pending' AND posted_at >= ?`, [this.importCutoff()], limit)
+    if (rows.length < limit) rows.push(...(await claim(`status = 'pending'`, [], limit - rows.length)))
+    if (rows.length < limit) {
+      rows.push(...(await claim(`status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}'`, [], limit - rows.length)))
+    }
     return rows
   }
 
-  /** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
+  /** Spots posted before this are listed, not imported: their NZB is read on first download. */
+  importCutoff(): Date {
+    return new Date(Date.now() - indexerConfig.spotImportDays * 24 * 60 * 60 * 1000)
+  }
+
+  /** Old, verified spots that aren't moderation: listed in bulk, without reading anything more. */
+  isListable(spot: any): boolean {
+    return (
+      spot.verified === true &&
+      spot.posted_at !== null &&
+      new Date(spot.posted_at) < this.importCutoff() &&
+      !moderationTarget({ keyId: spot.key_id, title: spot.title })
+    )
+  }
+
+  /**
+   * Creates the listing releases (source `spot`, no files) for these spots.
+   * The release key is the one an import of the spot's NZB uses, so the
+   * import on first download fills this release.
+   */
+  async list(spots: any[]): Promise<number> {
+    if (spots.length === 0) return 0
+    const keyOf = (spot: any) => createHash('sha1').update(`nzb\n${spot.message_id}`).digest('hex')
+    const ids = await IngestService.upsertReleases(
+      db.connection().getWriteClient(),
+      spots.map((spot) => ({
+        release: { key: keyOf(spot), name: spot.title, source: 'spot' as const },
+        poster: spot.poster,
+        groups: [spot.group_name],
+        postedAt: spot.posted_at ? new Date(spot.posted_at) : undefined,
+      }))
+    )
+    const rows = spots.map((spot) => ({ message_id: spot.message_id, release_id: ids.get(keyOf(spot)), size: spot.size }))
+    await db.rawQuery(
+      `UPDATE releases r SET size = v.size FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, size bigint)
+       WHERE r.id = v.release_id AND r.file_count = 0 AND v.size IS NOT NULL`,
+      [JSON.stringify(rows)]
+    )
+    await db.rawQuery(
+      `UPDATE spots s SET status = 'listed', release_id = v.release_id, processed_at = now()
+       FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, release_id bigint)
+       WHERE s.message_id = v.message_id`,
+      [JSON.stringify(rows)]
+    )
+    return rows.length
+  }
+
+  /**
+   * Verifies the spot on its raw header (the signature covers ISO-8859-1
+   * bytes, which the overview, read as UTF-8, may have mangled), then
+   * applies moderation, lists an old spot or imports a recent one's NZB.
+   * Returns the new status.
+   */
   async process(spot: any): Promise<string> {
     const update: Record<string, unknown> = { processed_at: new Date(), error: null }
     try {
-      const raw = await this.fetch(`HEAD <${spot.message_id}>`)
-      const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
-      // The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
-      const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
-      if (!header || !verifySpotHeader(header)) {
+      const head = await this.readHead(spot.message_id)
+      if (!head) {
         update.status = 'unverified'
-      } else if (moderationTarget(header)) {
+      } else if (moderationTarget(head.header)) {
         update.status = 'moderation'
-        await this.moderate(header, moderationTarget(header)!)
+        await this.moderate(head.header, moderationTarget(head.header)!)
       } else {
-        const parsed = parseSpotXml(xml)
-        update.title = header.title
-        update.poster = header.poster
-        update.spotter_id = spotterId(header.selfSignedPubKey) || null
-        if (parsed?.website) update.website = parsed.website
-        const segments = parsed?.nzbSegments ?? []
-        if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
-          update.status = 'no-nzb'
+        update.title = spotTitle(head)
+        update.poster = head.header.poster
+        update.spotter_id = spotterId(head.header.selfSignedPubKey) || null
+        Object.assign(update, this.details(head.xml))
+        if (new Date(head.header.stamp * 1000) < this.importCutoff()) {
+          await this.list([{ ...spot, title: spotTitle(head), poster: head.header.poster }])
+          update.status = 'listed'
         } else {
-          const bodies = []
-          for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
-          const nzbXml = decodeSpotBinary(bodies).toString('utf8')
-          const nzb = parseNzb(nzbXml)
-          if (!nzb || nzb.files.length === 0) {
-            update.status = 'no-nzb'
-          } else {
-            const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
-            const result = await NzbImportService.import(
-              {
-                id: null,
-                filename: `${header.title}.nzb`,
-                poster: header.poster,
-                groups,
-                date: header.stamp * 1000,
-                message_ids: { '1': { id: spot.message_id } },
-              },
-              nzbXml,
-              { name: header.title, source: 'spot', moveContainer: false }
-            )
-            update.status = result ? 'imported' : 'no-nzb'
-            if (result) update.release_id = result.releaseId
-          }
+          Object.assign(update, await this.importNzb(spot.message_id, head))
         }
       }
     } catch (error: any) {
@@ -131,9 +202,250 @@ class SpotService {
       logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
     }
     await db.from('spots').where('message_id', spot.message_id).update(update)
+    if (update.status === 'imported' || update.status === 'listed') await this.applyPassword(spot.message_id)
     return update.status as string
   }
 
+  /**
+   * The NZB of a listing, on first download: reads and imports it. Returns
+   * the release that now holds its files (the listing's own, unless the NZB
+   * joined files already known elsewhere), or null.
+   */
+  async importListing(releaseId: number): Promise<number | null> {
+    const spot = await db.from('spots').where('release_id', releaseId).where('status', 'listed').first()
+    if (!spot) return null
+    return this.reimport(spot)
+  }
+
+  /**
+   * Reads a spot's NZB again and imports it, so its files end up in the
+   * spot's own release under the spot's title. Returns that release, or null.
+   */
+  async reimport(spot: any): Promise<number | null> {
+    const head = await this.readHead(spot.message_id)
+    if (!head) return null
+    const result = await this.importNzb(spot.message_id, head)
+    await db
+      .from('spots')
+      .where('message_id', spot.message_id)
+      .update({ ...result, ...this.details(head.xml), processed_at: new Date() })
+    await this.applyPassword(spot.message_id)
+    return (result.release_id as number | undefined) ?? null
+  }
+
+  /**
+   * A password the spotter wrote in the spot's title or description (not in
+   * the NZB) goes on its release, unless the release has one already.
+   */
+  async applyPassword(messageId: string): Promise<string | null> {
+    const spot = await db.from('spots').where('message_id', messageId).whereNotNull('release_id').first()
+    const password = spot ? passwordFromSpot(spot.title, spot.description) : null
+    if (!password) return null
+    await db
+      .from('releases')
+      .where('id', spot.release_id)
+      .whereNull('password')
+      .update({ password, password_source: 'spot' })
+    return password
+  }
+
+  /** Columns for what the spot's XML adds to its overview: description, image, website. */
+  private details(xml: SpotXml | null): Record<string, unknown> {
+    return {
+      description: xml?.description || null,
+      image: xml?.image ? JSON.stringify(xml.image) : null,
+      ...(xml?.website ? { website: xml.website } : {}),
+      details_at: new Date(),
+    }
+  }
+
+  /**
+   * A spot row with its description and image, reading its full header
+   * first when that hasn't happened yet (old spots are listed from their
+   * overview only). Returns the row as it is when Usenet can't be reached.
+   */
+  async withDetails(spot: any): Promise<any> {
+    if (spot.details_at) return spot
+    try {
+      const head = await this.readHead(spot.message_id)
+      const update = head ? this.details(head.xml) : { details_at: new Date() }
+      await db.from('spots').where('message_id', spot.message_id).update(update)
+      await this.applyPassword(spot.message_id)
+      return { ...spot, ...update, image: head?.xml?.image ?? null }
+    } catch (error: any) {
+      logger.warn(`Spot ${spot.message_id} details: ${error.message}`)
+      // The article is gone (430): nothing to read later either.
+      if (/^4\d\d/.test(String(error.message))) {
+        await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
+      }
+      return spot
+    }
+  }
+
+  /**
+   * Reads the details of up to `limit` imported spots that don't have them
+   * (imported before details were kept), newest first. Returns how many.
+   * Listed spots are left to their first view: there are millions.
+   * Returns how many were read (0 when Usenet can't be reached).
+   */
+  async fillDetails(limit: number): Promise<number> {
+    const spots = await db
+      .from('spots')
+      .where('status', 'imported')
+      .whereNull('details_at')
+      .orderBy('posted_at', 'desc')
+      .limit(limit)
+    let read = 0
+    for (const spot of spots) if ((await this.withDetails(spot)).details_at) read++
+    return read
+  }
+
+  /**
+   * Takes up to `limit` listed spots whose article and NZB haven't been read,
+   * newest first, marking them so another reader skips them.
+   */
+  async claimListings(limit: number): Promise<any[]> {
+    const { rows } = await db.rawQuery(
+      `UPDATE spots SET nzb_read_at = now()
+       WHERE message_id IN (
+         SELECT message_id FROM spots WHERE status = 'listed' AND nzb_read_at IS NULL
+         ORDER BY posted_at DESC LIMIT ? FOR UPDATE SKIP LOCKED)
+       RETURNING *`,
+      [limit]
+    )
+    return rows
+  }
+
+  /**
+   * Reads a listed spot's article and NZB without importing its files
+   * (LOG.md 2026-10-08 (10)): the details (description, image), the NZB's
+   * own password (`password_source = 'nzb'`, over one from the spot's text),
+   * else one in the spot's text; a placeholder NZB hides the spot (`dummy`).
+   * Returns what it found. A connection failure gives the spot back for later.
+   */
+  async readListing(spot: any): Promise<string> {
+    try {
+      const head = await this.readHead(spot.message_id)
+      if (!head) {
+        await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
+        return 'unverified'
+      }
+      const update: Record<string, unknown> = this.details(head.xml)
+      let outcome = 'nothing'
+      let nzbXml: string | null = null
+      try {
+        nzbXml = await this.readNzb(head)
+      } catch (error: any) {
+        if (error instanceof NzbUnreadable) {
+          // Corrupt: reading it again won't help (LOG.md 2026-10-08 (12)).
+          update.error = `NZB unreadable: ${error.message}`.slice(0, 500)
+          outcome = 'nzb-unreadable'
+        } else if (/^4\d\d/.test(String(error.message))) {
+          // The NZB's articles are gone: the listing can't be downloaded any more.
+          update.error = 'NZB articles gone'
+          outcome = 'nzb-gone'
+        } else {
+          throw error
+        }
+      }
+      const nzb = nzbXml ? parseNzb(nzbXml) : null
+      if (nzb && nzb.files.length > 0 && isDummyNzb(nzb, head.header.size)) {
+        update.status = 'dummy'
+        outcome = 'dummy'
+      } else if (nzb?.meta.password && spot.release_id) {
+        await db
+          .from('releases')
+          .where('id', spot.release_id)
+          .where((q) => q.whereNull('password').orWhere('password_source', 'spot'))
+          .update({ password: nzb.meta.password, password_source: 'nzb' })
+        outcome = 'nzb-password'
+      }
+      await db.from('spots').where('message_id', spot.message_id).update(update)
+      if (outcome !== 'dummy' && (await this.applyPassword(spot.message_id)) && outcome !== 'nzb-password') outcome = 'spot-password'
+      return outcome
+    } catch (error: any) {
+      // The spot's own article is gone (430): nothing to read later either.
+      if (/^4\d\d/.test(String(error.message))) {
+        await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
+        return 'gone'
+      }
+      await db.from('spots').where('message_id', spot.message_id).update({ nzb_read_at: null })
+      logger.warn(`Spot ${spot.message_id} listing: ${error.message}`)
+      return 'retry'
+    }
+  }
+
+  /** Reads up to `limit` unread listings, `concurrency` at a time. Returns the outcomes counted. */
+  async readListings(limit: number, concurrency: number): Promise<Record<string, number>> {
+    const spots = await this.claimListings(limit)
+    const counts: Record<string, number> = {}
+    let next = 0
+    await Promise.all(
+      Array.from({ length: Math.max(1, concurrency) }, async () => {
+        while (next < spots.length) {
+          const outcome = await this.readListing(spots[next++])
+          counts[outcome] = (counts[outcome] ?? 0) + 1
+        }
+      })
+    )
+    return counts
+  }
+
+  /** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
+  async image(spot: any): Promise<Buffer | null> {
+    const segments: string[] = spot.image?.segments ?? []
+    if (segments.length === 0 || segments.length > MAX_IMAGE_SEGMENTS) return null
+    const bodies = []
+    for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
+    return decodeSpotBinary(bodies, false)
+  }
+
+  /** The spot's raw header, verified; null when it doesn't verify. */
+  private async readHead(messageId: string): Promise<{ header: SpotHeader; xml: SpotXml | null } | null> {
+    const raw = await this.fetch(`HEAD <${messageId}>`)
+    const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
+    const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
+    if (!header || !verifySpotHeader(header)) return null
+    return { header, xml: parseSpotXml(decodeSpotText(xml)) }
+  }
+
+  /** Downloads and decodes the spot's compressed NZB; null when it has none (or a too large one). */
+  private async readNzb(head: { xml: SpotXml | null }): Promise<string | null> {
+    const segments = head.xml?.nzbSegments ?? []
+    if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return null
+    const bodies = []
+    for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
+    try {
+      return decodeSpotBinary(bodies).toString('utf8')
+    } catch (error: any) {
+      throw new NzbUnreadable(String(error.message))
+    }
+  }
+
+  /** Downloads the spot's compressed NZB and imports it as a release (D26). */
+  private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
+    const nzbXml = await this.readNzb(head)
+    if (!nzbXml) return { status: 'no-nzb' }
+    const nzb = parseNzb(nzbXml)
+    if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
+    const { header } = head
+    if (isDummyNzb(nzb, header.size)) return { status: 'dummy' }
+    const title = spotTitle(head)
+    const result = await NzbImportService.import(
+      {
+        id: null,
+        filename: `${title}.nzb`,
+        poster: header.poster,
+        groups: [...new Set(nzb.files.flatMap((f) => f.groups))],
+        date: header.stamp * 1000,
+        first_message_id: messageId,
+      },
+      nzbXml,
+      { name: title, source: 'spot', moveContainer: false, trusted: true }
+    )
+    return result ? { status: 'imported', release_id: result.releaseId } : { status: 'no-nzb' }
+  }
+
   /**
    * Applies a delete/dispose/remove: the target spot is marked `removed`
    * (its release is then hidden from search). As in Spotweb, a personal
@@ -150,6 +462,15 @@ class SpotService {
     }
     await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
     logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
+    // Spots that joined its release (a correction, a repost) take the files over now.
+    if (target.release_id) {
+      const sharing = await db
+        .from('spots')
+        .where('release_id', target.release_id)
+        .where('status', 'imported')
+        .whereNot('message_id', targetId)
+      for (const spot of sharing) await this.reimport(spot)
+    }
   }
 
   /** One HEAD or BODY, as raw bytes. */
@@ -157,7 +478,7 @@ class SpotService {
     const conn = await NntpService.acquire()
     let failed = false
     try {
-      const { data } = await conn._artcmd(command)
+      const { data } = await NntpService.timed<any>(conn._artcmd(command), command)
       return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
     } catch (error: any) {
       // 430 (no such article) is an answer, not a broken connection.

app/services/StagingService.ts

@@ -0,0 +1,252 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import indexerConfig from '#config/indexer'
+import IngestService, { type IngestSegment } from '#services/IngestService'
+import { counterFileKey } from '#lib/subject/index'
+import { fileStem } from '#lib/release/index'
+
+/** Rows per statement (a key and a JSON array each). */
+const CHUNK = 5000
+
+/**
+ * Staging of lone header segments (CONCEPT.md D35). Uploads that give every
+ * article its own random subject turned each article into a binary, a part
+ * and a release, which GC deleted two hours later: ~3.9M binaries an hour
+ * against ~123k real files. A segment whose file is new and alone in its
+ * fetch batch now waits in `staged_segments` (unlogged, one index) until a
+ * second segment of that file arrives; only then do both go into the real
+ * tables. GC drops what waited longer than STAGING_TTL_SECONDS (30 min).
+ *
+ * Counter grouping (D36): those random-subject uploads still keep poster,
+ * file counter `[i/M]` and part count per file. Staged segments also carry
+ * that key (`counter_key`); once two staged segments from different subjects
+ * share it, they become one file under that key, and later segments with it
+ * join that file directly.
+ */
+class StagingService {
+  enabled(groupName: string): boolean {
+    const groups = indexerConfig.stagingGroups
+    return groups.includes('*') || groups.includes(groupName)
+  }
+
+  /** Whether counter grouping (D36) applies to this group. */
+  countersEnabled(groupName: string): boolean {
+    const groups = indexerConfig.counterGroupingGroups
+    return groups.includes('*') || groups.includes(groupName)
+  }
+
+  /**
+   * The segments of a header batch that should be ingested now: the batch's
+   * own, minus lone segments of new files (staged instead), plus staged
+   * segments whose file this batch shows to be real. `staged` counts the
+   * segments that went to staging.
+   */
+  async filter(segments: IngestSegment[], { counters = false } = {}): Promise<{ ingest: IngestSegment[]; staged: number }> {
+    const byKey = new Map<string, IngestSegment[]>()
+    for (const segment of segments) {
+      const list = byKey.get(segment.fileKey)
+      if (list) list.push(segment)
+      else byKey.set(segment.fileKey, [segment])
+    }
+
+    const ingest: IngestSegment[] = []
+    const lone: IngestSegment[] = []
+    for (const list of byKey.values()) {
+      // Single-segment files are complete; two different articles of a file in one batch make it real.
+      // (One article twice is a crosspost, LOG.md 2026-10-02.)
+      const articles = new Set(list.map((s) => s.messageId)).size
+      if (list[0].totalParts <= 1 || articles >= 2 || list[0].weakIdentity || list[0].size != null) ingest.push(...list)
+      else lone.push(list[0])
+    }
+
+    const counterOf = new Map<IngestSegment, string>()
+    if (counters) {
+      for (const segment of lone) {
+        const key = counterFileKey(segment)
+        if (key) counterOf.set(segment, key)
+      }
+    }
+
+    // Files we already have a binary for (by subject, or by counters) take their segments directly.
+    const known = new Set<string>()
+    const lookup = [...new Set([...lone.map((s) => s.fileKey), ...counterOf.values()])]
+    for (let i = 0; i < lookup.length; i += CHUNK) {
+      const rows = await db.from('binaries').whereIn('file_key', lookup.slice(i, i + CHUNK)).select('file_key')
+      for (const row of rows) known.add(row.file_key)
+    }
+    const toStage: IngestSegment[] = []
+    const knownCounters = new Set<string>()
+    for (const segment of lone) {
+      const counter = counterOf.get(segment)
+      if (known.has(segment.fileKey)) {
+        ingest.push(segment)
+      } else if (counter && known.has(counter)) {
+        ingest.push(asCounterFile(segment, counter))
+        knownCounters.add(counter)
+      } else {
+        toStage.push(segment)
+      }
+    }
+
+    // Stage the rest; a file that now has two staged segments comes out again.
+    const promoted: string[] = []
+    // Key order, so concurrent jobs lock rows in the same order.
+    toStage.sort((a, b) => a.fileKey.localeCompare(b.fileKey))
+    for (let i = 0; i < toStage.length; i += CHUNK) {
+      const rows = toStage.slice(i, i + CHUNK).map((s) => ({
+        file_key: s.fileKey,
+        counter_key: counterOf.get(s) ?? null,
+        segments: [serialize(s)],
+        staged_at: new Date(),
+      }))
+      const { rows: result } = await retryDeadlock(() => db.rawQuery(
+        `INSERT INTO staged_segments (file_key, counter_key, segments, staged_at)
+         SELECT file_key, counter_key, segments, staged_at
+         FROM jsonb_to_recordset(?::jsonb) AS v(file_key text, counter_key text, segments jsonb, staged_at timestamptz)
+         ON CONFLICT (file_key) DO UPDATE SET segments = CASE
+           -- The same article again (a crosspost, or a re-fetched range) is no second segment.
+           WHEN EXISTS (SELECT 1 FROM jsonb_array_elements(staged_segments.segments) e
+                        WHERE e->>'messageId' = EXCLUDED.segments->0->>'messageId') THEN staged_segments.segments
+           ELSE staged_segments.segments || EXCLUDED.segments END
+         RETURNING file_key, jsonb_array_length(segments) AS n`,
+        [JSON.stringify(rows)]
+      ))
+      for (const row of result) if (Number(row.n) >= 2) promoted.push(row.file_key)
+    }
+
+    // Counter files: two staged rows (two subjects) with one counter key make a file.
+    const stagedCounters = [...new Set(toStage.map((s) => counterOf.get(s)).filter((k): k is string => Boolean(k)))]
+    const readyCounters: string[] = []
+    for (let i = 0; i < stagedCounters.length; i += CHUNK) {
+      const { rows } = await db.rawQuery(
+        `SELECT counter_key FROM staged_segments WHERE counter_key = ANY(?) GROUP BY counter_key HAVING count(*) >= 2`,
+        [stagedCounters.slice(i, i + CHUNK)]
+      )
+      for (const row of rows) readyCounters.push(row.counter_key)
+    }
+
+    // Promoted files, staged segments of files this batch ingests anyway, and ready counter files.
+    const fileKeys = [...new Set([...promoted, ...ingest.map((s) => s.fileKey)])]
+    const counterKeys = [...new Set([...readyCounters, ...knownCounters])]
+    const seen = new Set(ingest.map((s) => s.messageId))
+    for (const segment of await this.take(fileKeys, counterKeys)) {
+      if (!seen.has(segment.messageId)) ingest.push(segment)
+      seen.add(segment.messageId)
+    }
+    const staged = toStage.filter((s) => !seen.has(s.messageId)).length
+    return { ingest, staged }
+  }
+
+  /**
+   * GC: drops staged segments that waited longer than `maxAgeSeconds`, then
+   * ingests those whose file got a binary meanwhile (two fetch jobs racing).
+   * Expiry comes first so the table stays bounded whatever the budget; the
+   * adoption starts from recently seen binaries (index on last_seen_at,
+   * which moves in 10-minute steps) rather than joining every staged row.
+   */
+  async sweep(maxAgeSeconds: number, budget: number): Promise<{ adopted: number; expired: number }> {
+    let adopted = 0
+    let expired = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM staged_segments WHERE ctid IN (
+           SELECT ctid FROM staged_segments WHERE staged_at < now() - make_interval(secs => ?) LIMIT 20000)`,
+        [maxAgeSeconds]
+      )
+      expired += result.rowCount
+      if (result.rowCount < 20000) break
+    }
+    let lastId = 0
+    while (Date.now() < budget) {
+      const recent = await db
+        .from('binaries')
+        .where('last_seen_at', '>', new Date(Date.now() - (maxAgeSeconds + 15 * 60) * 1000))
+        .where('id', '>', lastId)
+        .orderBy('id')
+        .limit(CHUNK)
+        .select('id', 'file_key')
+      if (recent.length === 0) break
+      lastId = Number(recent[recent.length - 1].id)
+      const keys = recent.map((b) => b.file_key as string)
+      // A binary's key is a subject key or a counter key; staged rows are looked up by both.
+      const segments = await this.take(keys, keys)
+      if (segments.length > 0) await IngestService.ingest(segments)
+      adopted += segments.length
+    }
+    return { adopted, expired }
+  }
+
+  /**
+   * Removes and returns the staged segments of these files: by subject key,
+   * or by counter key (returned as segments of the counter file).
+   */
+  private async take(fileKeys: string[], counterKeys: string[] = []): Promise<IngestSegment[]> {
+    const taken: IngestSegment[] = []
+    for (let i = 0; i < fileKeys.length; i += CHUNK) {
+      const { rows } = await retryDeadlock(() =>
+        db.rawQuery(`DELETE FROM staged_segments WHERE file_key = ANY(?) RETURNING segments`, [fileKeys.slice(i, i + CHUNK)])
+      )
+      for (const row of rows) taken.push(...(row.segments as any[]).map(deserialize))
+    }
+    for (let i = 0; i < counterKeys.length; i += CHUNK) {
+      const { rows } = await retryDeadlock(() =>
+        db.rawQuery(`DELETE FROM staged_segments WHERE counter_key = ANY(?) RETURNING counter_key, segments`, [counterKeys.slice(i, i + CHUNK)])
+      )
+      for (const row of rows) taken.push(...(row.segments as any[]).map((s) => asCounterFile(deserialize(s), row.counter_key)))
+    }
+    return taken
+  }
+}
+
+/**
+ * A segment as part of its counter file (D36): the counter key is its file
+ * key, it is a weak identity (a repeated part number from another upload
+ * marks the binary conflicted), and its release is a weak one of its own,
+ * which the upload's par2 set can gather later.
+ */
+function asCounterFile(segment: IngestSegment, counterKey: string): IngestSegment {
+  return {
+    ...segment,
+    fileKey: counterKey,
+    weakIdentity: true,
+    release: {
+      key: createHash('sha1').update(`counter-release\n${counterKey}`).digest('hex'),
+      name: fileStem(segment.name),
+      source: 'weak-stem',
+    },
+  }
+}
+
+/**
+ * Runs a staging statement again when it lost a deadlock: concurrent fetch
+ * jobs insert by file key and delete by counter key, so their row locks can
+ * cross. Postgres rolled the statement back, and each one is safe to repeat.
+ */
+async function retryDeadlock<T>(statement: () => Promise<T>, attempts = 5): Promise<T> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await statement()
+    } catch (error: any) {
+      if (error?.code !== '40P01' || attempt >= attempts) throw error
+      await new Promise((resolve) => setTimeout(resolve, 50 * attempt + Math.random() * 100))
+    }
+  }
+}
+
+function serialize(segment: IngestSegment) {
+  return {
+    ...segment,
+    postedAt: segment.postedAt?.toISOString(),
+    articleNumber: segment.articleNumber?.toString(),
+  }
+}
+
+function deserialize(stored: any): IngestSegment {
+  return {
+    ...stored,
+    postedAt: stored.postedAt ? new Date(stored.postedAt) : undefined,
+    articleNumber: stored.articleNumber ?? undefined,
+  }
+}
+
+export default new StagingService()

bin/console.ts

@@ -18,6 +18,12 @@ const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
  * URL to the application root. AdonisJS need it to resolve
  * paths to file and directories for scaffolding commands
  */
+/**
+ * The command being run, for start/env.ts: Ace takes the arguments out of
+ * process.argv before the environment is loaded.
+ */
+process.env.ACE_COMMAND ??= process.argv[2] ?? ''
+
 const APP_ROOT = new URL('../', import.meta.url)
 
 /**

commands/ArchiveWalk.ts

@@ -0,0 +1,70 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import QueueService from '#services/QueueService'
+import NntpService from '#services/NntpService'
+import ArchiveWalkService from '#services/ArchiveWalkService'
+
+const BATCH = 5000
+
+/**
+ * Complete archive listings for releases inspected before archive walks
+ * existed (CONCEPT.md D40): queues those whose listing leaves most of the
+ * release unaccounted for. worker:collection walks them, one at a time and
+ * behind inspection. With release ids, walks those right here instead.
+ */
+export default class ArchiveWalk extends BaseCommand {
+  public static commandName = 'content:walk-archives'
+  public static description = 'Queues releases with partial archive listings for a (low priority) header walk.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Release ids to walk now (instead of queueing)', required: false })
+  declare releaseIds?: string[]
+
+  @flags.number({ description: 'Queue at most this many releases' })
+  declare limit?: number
+
+  @flags.boolean({ description: 'Also releases that were walked before' })
+  declare again: boolean
+
+  public async run() {
+    try {
+      if (this.releaseIds?.length) {
+        for (const id of this.releaseIds) {
+          this.logger.info(JSON.stringify(await ArchiveWalkService.walk(Number(id))))
+        }
+        return
+      }
+      let lastId = 0
+      let queued = 0
+      while (!this.limit || queued < this.limit) {
+        const query = db
+          .from('releases as r')
+          .where('r.id', '>', lastId)
+          .whereNotNull('r.contents')
+          .orderBy('r.id')
+          .limit(BATCH)
+          .select('r.id', 'r.size', 'r.contents_walked_at', db.raw(`(
+            SELECT count(*) FROM jsonb_array_elements(r.contents)) AS n`), db.raw(`(
+            SELECT coalesce(sum((e->>'size')::bigint), 0) FROM jsonb_array_elements(r.contents) e) AS listed`))
+        const rows = await query
+        if (rows.length === 0) break
+        lastId = Number(rows.at(-1).id)
+        const jobs = rows
+          .filter((r) => this.again || !r.contents_walked_at)
+          .filter((r) => ArchiveWalkService.worthWalking(Number(r.n), Number(r.listed), Number(r.size)))
+          .slice(0, this.limit ? this.limit - queued : undefined)
+          .map((r) => ({ name: 'walk', data: { releaseId: Number(r.id) }, opts: { jobId: `walk-${r.id}-${Date.now()}` } }))
+        if (jobs.length > 0) await QueueService.archiveWalkQueue.addBulk(jobs)
+        queued += jobs.length
+      }
+      this.logger.info(`Queued ${queued} releases for an archive walk.`)
+    } finally {
+      await QueueService.closeAll()
+      await NntpService.shutdown()
+    }
+  }
+}

commands/ContentBackfill.ts

@@ -3,6 +3,8 @@ import { CommandOptions } from '@adonisjs/core/types/ace'
 import db from '@adonisjs/lucid/services/db'
 import QueueService from '#services/QueueService'
 import ContentService from '#services/ContentService'
+import { isArchiveStart } from '#lib/content/index'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
 
 const BATCH = 1000
 
@@ -28,18 +30,38 @@ export default class ContentBackfill extends BaseCommand {
   @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
   declare archiveNzbs: boolean
 
+  @flags.boolean({ description: 'Only unlisted 7z archives (first volumes: .7z, .7z.001)' })
+  declare sevenZip: boolean
+
+  @flags.boolean({ description: 'Archive starts never inspected (RAR first volumes, 7z, .bin), also files from NZBs/spots' })
+  declare archives: boolean
+
+  @flags.boolean({ description: 'SFV files not read yet (their listings group the files they name, D34)' })
+  declare sfv: boolean
+
+  @flags.boolean({ description: 'niconico captures not read yet (their releases get the video title, D34)' })
+  declare nicovideo: boolean
+
   public async run() {
     let lastId = 0
     let queued = 0
     try {
       while (!this.limit || queued < this.limit) {
         // Files never inspected: no fingerprint on their binary and no archive listing.
-        const query = db
-          .from('files as f')
-          .join('binaries as b', 'b.file_id', 'f.id')
-          .where('f.id', '>', lastId)
-        if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
-        else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
+        const query = db.from('files as f').leftJoin('file_details as d', 'd.file_id', 'f.id').where('f.id', '>', lastId)
+        // Files from NZBs (spots) have no binary.
+        if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
+        else query.join('binaries as b', 'b.file_id', 'f.id')
+        if (this.nicovideo) {
+          query.whereNull('d.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
+        } else if (this.sfv) {
+          query.whereNull('d.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
+        } else if (this.archives) {
+          query.whereNull('d.content_type').whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
+        } else if (this.sevenZip) {
+          query.whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
+        } else if (this.archiveNzbs) query.whereRaw(`d.archive::text ~* '\\.nzb"'`)
+        else if (!this.all) query.whereNull('b.md5_16k').whereNull('d.archive')
         const files = await query
           .orderBy('f.id')
           .limit(BATCH)
@@ -48,16 +70,17 @@ export default class ContentBackfill extends BaseCommand {
         lastId = Number(files.at(-1).id)
 
         const jobs = files
-          .filter((f) => ContentService.isCandidate(f.filename))
+          .filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
           .slice(0, this.limit ? this.limit - queued : undefined)
-          .map((f) => ({
-            name: 'process-collection',
-            data: { fileId: Number(f.id) },
-            // A new id per run: BullMQ ignores an add() for an id it still knows.
-            // Targeted backfills go ahead of whatever is waiting.
-            opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
-          }))
-        if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+          .map((f) => Number(f.id))
+        // Into the inspection backlog (InspectionQueue): targeted backfills go ahead of what's waiting;
+        // niconico captures are many (~190k) and only cosmetic, so behind everything else.
+        const priority = this.nicovideo
+          ? INSPECTION_PRIORITY.low
+          : this.archiveNzbs || this.sevenZip || this.archives || this.sfv
+            ? INSPECTION_PRIORITY.targeted
+            : INSPECTION_PRIORITY.normal
+        if (jobs.length > 0) await InspectionQueue.add(jobs, priority)
         queued += jobs.length
       }
       this.logger.info(`Queued ${queued} files for content inspection.`)

commands/DbReindex.ts

@@ -0,0 +1,22 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import IndexMaintenance from '#services/IndexMaintenance'
+
+/** Rebuilds bloated indexes online (gc:scheduler also does this weekly). */
+export default class DbReindex extends BaseCommand {
+  public static commandName = 'db:reindex'
+  public static description = 'Rebuilds B-tree indexes that are mostly empty space (online, REINDEX CONCURRENTLY).'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show which indexes would be rebuilt' })
+  declare dryRun: boolean
+
+  public async run() {
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`${rebuilt.length} indexes ${this.dryRun ? 'would be rebuilt' : `rebuilt, ${Math.round(saved / 1024 / 1024)} MB freed`}.`)
+  }
+}

commands/FilesCleanNames.ts

@@ -0,0 +1,35 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { unquotedFileName } from '#services/NzbImportService'
+
+/**
+ * Files named after a whole NZB subject without quotes (spot NZBs), e.g.
+ * `Adventure.part0055.rar - [056/175] (1/1)`: renamed to the filename in it.
+ */
+export default class FilesCleanNames extends BaseCommand {
+  public static commandName = 'files:clean-names'
+  public static description = 'Strips subject leftovers (counters, "yEnc") from file names that came from NZBs.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show what would change' })
+  declare dryRun: boolean
+
+  public async run() {
+    const files = await db
+      .from('files')
+      .whereRaw(`filename ~* '(\\(\\d+/\\d+\\)|yEnc|\\[\\d+/\\d+\\])\\s*$'`)
+      .select('id', 'filename')
+    const changes = files
+      .map((f) => ({ id: Number(f.id), from: f.filename as string, to: unquotedFileName(f.filename) }))
+      .filter((c) => c.to !== c.from)
+    for (const c of changes.slice(0, 5)) this.logger.info(`"${c.from}" → "${c.to}"`)
+    if (!this.dryRun) {
+      for (const c of changes) await db.from('files').where('id', c.id).update({ filename: c.to })
+    }
+    this.logger.info(`${changes.length} of ${files.length} file names ${this.dryRun ? 'would change' : 'cleaned'}.`)
+  }
+}

commands/FilesMoveDetails.ts

@@ -0,0 +1,36 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+
+/**
+ * One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
+ * listings still on `files` into file_details, a partition at a time.
+ * Values written since the deploy (by the new code, into file_details) win:
+ * a file that has details already is skipped.
+ */
+export default class FilesMoveDetails extends BaseCommand {
+  public static commandName = 'files:move-details'
+  public static description = "Copies files' content types and archive listings into file_details."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const { rows: partitions } = await db.rawQuery(
+      `SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname`
+    )
+    let total = 0
+    for (const { relname } of partitions) {
+      const started = Date.now()
+      const result = await db.rawQuery(
+        `INSERT INTO file_details (file_id, content_type, archive)
+         SELECT id, content_type, archive FROM ${relname} WHERE content_type IS NOT NULL OR archive IS NOT NULL
+         ON CONFLICT (file_id) DO NOTHING`
+      )
+      total += result.rowCount ?? 0
+      this.logger.info(`${relname}: ${result.rowCount} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
+    }
+    this.logger.info(`${total} files' details copied.`)
+  }
+}

commands/GarbageLearn.ts

@@ -0,0 +1,38 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import GarbageFilter from '#services/GarbageFilter'
+
+/**
+ * Learns the garbage upload signatures now (gc:scheduler does it every
+ * GARBAGE_LEARN_INTERVAL_MS) and lists them (CONCEPT.md D42). --seed first
+ * fills signature_probes from a sample of the files probed before it
+ * existed, so there is something to learn from right away.
+ */
+export default class GarbageLearn extends BaseCommand {
+  public static commandName = 'garbage:learn'
+  public static description = 'Learns which upload signatures are garbage and lists them.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'First add probed releases from TABLESAMPLE SYSTEM (percent) of files' })
+  declare seed?: number
+
+  public async run() {
+    if (this.seed) {
+      const started = Date.now()
+      const added = await GarbageFilter.seed(this.seed)
+      this.logger.info(`Seeded ${added} probed releases from ${this.seed}% of files in ${Math.round((Date.now() - started) / 1000)} s.`)
+    }
+    const { signatures, dropped, changed } = await GarbageFilter.learn()
+    for (const line of changed) this.logger.info(`Changed: ${line}`)
+    const rows = await db.from('upload_signatures').orderBy('probed', 'desc').limit(40)
+    for (const row of rows) {
+      const rate = row.probed ? ((100 * row.named) / row.probed).toFixed(1) : '-'
+      this.logger.info(`${row.dropped ? 'DROP' : 'keep'}  ${row.tool.padEnd(10)} ${(row.group_name || '(no group)').padEnd(36)} ${String(row.named).padStart(6)}/${String(row.probed).padEnd(6)} named (${rate}%)`)
+    }
+    this.logger.info(`${dropped} of ${signatures} signatures dropped.`)
+  }
+}

commands/GcScheduler.ts

@@ -5,10 +5,20 @@ import db from '@adonisjs/lucid/services/db'
 import gcConfig from '#config/gc'
 import QueueService from '#services/QueueService'
 import RedisService from '#services/RedisService'
+import IndexMaintenance from '#services/IndexMaintenance'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+import StagingService from '#services/StagingService'
+import FilePartitions from '#services/FilePartitions'
+import ReleaseCompleteness from '#services/ReleaseCompleteness'
+import FileKeys from '#services/FileKeys'
+import GarbageFilter from '#services/GarbageFilter'
+import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
 
 const CLEAN_BATCH = 1000
 const SCAN_COUNT = 1000
+/** Binaries per statement when dropping copied parts (a few hundred parts each at most). */
+const COPIED_PARTS_BATCH = 2000
 
 export default class GcScheduler extends BaseCommand {
   public static commandName = 'gc:scheduler'
@@ -24,6 +34,13 @@ export default class GcScheduler extends BaseCommand {
   @flags.boolean({ description: 'Report what would be removed without changing anything' })
   declare dryRun: boolean
 
+  /** When release completeness was last recomputed (it runs less often than the sweep). */
+  private completenessAt = 0
+  /** When the garbage signatures were last learned (D42). */
+  private learnedAt = 0
+  /** When split uploads were last merged. */
+  private mergedAt = 0
+
   public async run() {
     this.logger.info(
       `GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
@@ -36,6 +53,21 @@ export default class GcScheduler extends BaseCommand {
         } catch (err: any) {
           this.logger.error(`GC sweep failed: ${err.message}`)
         }
+        try {
+          await this.releaseCompleteness()
+        } catch (err: any) {
+          this.logger.error(`Release completeness failed: ${err.message}`)
+        }
+        try {
+          await this.fillFileKeys()
+        } catch (err: any) {
+          this.logger.error(`file_keys fill failed: ${err.message}`)
+        }
+        try {
+          await this.learnGarbage()
+        } catch (err: any) {
+          this.logger.error(`Garbage signatures failed: ${err.message}`)
+        }
         if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
       } while (!this.once && !Shutdown.requested)
     } finally {
@@ -56,6 +88,16 @@ export default class GcScheduler extends BaseCommand {
       }
     }
 
+    // Inserts fail without a partition for their id (D39). Four are kept ahead, so a failure here
+    // can wait for the next sweep; it used to end the sweep, deletes and all (LOG.md 2026-10-07 (4)).
+    if (!this.dryRun) {
+      try {
+        await FilePartitions.ensure()
+      } catch (err: any) {
+        this.logger.error(`files partitions: ${err.message}; trying again next sweep.`)
+      }
+    }
+
     const trimmed = await this.trimBodyQueue()
     if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`)
 
@@ -63,13 +105,20 @@ export default class GcScheduler extends BaseCommand {
     this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
 
     if (!this.dryRun) {
-      const budget = Date.now() + gcConfig.deleteBudgetMs
-      const parts = await this.dropCopiedParts(budget)
-      const single = await this.deleteSingleArticleBinaries(budget)
-      const releases = await this.deleteEmptyReleases(budget)
-      if (parts || single || releases) {
+      // Each kind gets its own time budget; empty releases last, as deleting binaries empties them.
+      const budget = () => Date.now() + gcConfig.deleteBudgetMs
+      const parts = await this.dropCopiedParts(budget())
+      const single = await this.deleteSingleArticleBinaries(budget())
+      const unfingerprinted = await this.deleteUnfingerprintedBinaries(budget())
+      const releases = await this.deleteEmptyReleases(budget())
+      const staging = await StagingService.sweep(gcConfig.stagingTtlSeconds, budget())
+      if (staging.adopted || staging.expired) {
+        this.logger.info(`Staging: ${staging.adopted} segments adopted by their file, ${staging.expired} expired.`)
+      }
+      if (parts || single || unfingerprinted || releases) {
         this.logger.info(
-          `Space: ${parts} copied parts, ${single} single-article binaries, ${releases} empty releases deleted.`
+          `Space: ${parts} copied parts, ${single} single-article binaries, ${unfingerprinted} unfingerprinted completed binaries, ` +
+            `${releases} empty releases deleted.`
         )
       }
     }
@@ -83,6 +132,55 @@ export default class GcScheduler extends BaseCommand {
         `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
       )
     }
+
+    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep. It reads
+    // every release and its files: hourly, not every sweep (LOG.md 2026-10-06 (12)).
+    if (this.once || Date.now() - this.mergedAt >= gcConfig.mergeIntervalMs) {
+      this.mergedAt = Date.now()
+      const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+      if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
+    }
+
+    await this.rebuildIndexes()
+  }
+
+  /** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
+  /** The keys of files from before file_keys, a slice of time per sweep, until done. */
+  private async fillFileKeys() {
+    if (this.dryRun || (await FileKeys.complete())) return
+    const started = Date.now()
+    const { read, complete, nextId, untilId } = await FileKeys.fill(gcConfig.fileKeysFillMs)
+    this.logger.info(
+      `file_keys: ${read.toLocaleString('en')} keys copied in ${Math.round((Date.now() - started) / 1000)} s, ` +
+        (complete ? 'complete: file inserts use the filter now.' : `at file ${nextId.toLocaleString('en')} of ${untilId.toLocaleString('en')}.`)
+    )
+  }
+
+  private async releaseCompleteness() {
+    if (this.dryRun || (!this.once && Date.now() - this.completenessAt < gcConfig.releaseCompleteIntervalMs)) return
+    this.completenessAt = Date.now()
+    const started = Date.now()
+    const checked = await ReleaseCompleteness.sweep()
+    if (checked > 0) this.logger.info(`Release completeness: ${checked} changed releases checked in ${Math.round((Date.now() - started) / 1000)} s.`)
+  }
+
+  /** Learns which upload signatures are garbage from the probed releases (CONCEPT.md D42). */
+  private async learnGarbage() {
+    if (this.dryRun || (!this.once && Date.now() - this.learnedAt < indexerConfig.garbage.learnIntervalMs)) return
+    this.learnedAt = Date.now()
+    const { signatures, dropped, changed } = await GarbageFilter.learn()
+    this.logger.info(`Garbage signatures: ${dropped} of ${signatures} dropped.`)
+    for (const line of changed) this.logger.info(`Garbage signature ${line}`)
+  }
+
+  private async rebuildIndexes() {
+    const key = 'gc:reindex:last'
+    const last = Number((await RedisService.client.get(key)) ?? 0)
+    if (Date.now() - last < gcConfig.reindexIntervalMs) return
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    if (!this.dryRun) await RedisService.client.set(key, String(Date.now()))
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`Indexes: ${this.verb('rebuild', 'rebuilt')} ${rebuilt.length}, ${Math.round(saved / 1024 / 1024)} MB freed.`)
   }
 
   /**
@@ -111,19 +209,27 @@ export default class GcScheduler extends BaseCommand {
    * already live on in `files`; conflicted ones will never become a file.
    * Completed ones still waiting for worker:file are kept.
    */
+  /**
+   * Binaries without new segments for staleAfterSeconds: incomplete ones are given up on,
+   * finished ones have their file. A finished one whose file waits for content inspection
+   * stays until it's inspected (its fingerprint names it from par2 sets, D16), at most
+   * awaitInspectionSeconds (LOG.md 2026-10-06 (11)). Three deletes, one per partial index,
+   * so none reads past millions of waiting rows.
+   */
   private async deleteStaleBinaries(): Promise<number> {
-    const query = db
-      .from('binaries')
-      .where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
-      .where((q) =>
-        q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
-      )
-
-    if (this.dryRun) {
-      const [{ count }] = await query.count('* as count')
-      return Number(count)
+    const stale = new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000)
+    const waited = new Date(Date.now() - gcConfig.binaries.awaitInspectionSeconds * 1000)
+    const deletes = [
+      db.from('binaries').where('last_seen_at', '<', stale).whereNotNull('file_id').whereRaw('awaiting_inspection IS NOT TRUE'),
+      db.from('binaries').where('last_seen_at', '<', stale).where((q) => q.whereNull('completed_at').orWhereNotNull('conflicted_at')),
+      db.from('binaries').where('last_seen_at', '<', waited).whereRaw('awaiting_inspection IS TRUE'),
+    ]
+    let total = 0
+    for (const query of deletes) {
+      if (this.dryRun) total += Number((await query.count('* as count'))[0].count)
+      else total += Number(await query.delete())
     }
-    return Number(await query.delete())
+    return total
   }
 
   private async deleteOldDeferredFiles(): Promise<number> {
@@ -146,20 +252,32 @@ export default class GcScheduler extends BaseCommand {
   private singleCursor = 0
   private releaseCursor = 0
 
-  /** Parts of header-derived binaries that are a file already (worker:file now drops them itself). */
+  /**
+   * Header parts of binaries that are a file already. worker:file drops them
+   * when it creates the file, and ingest no longer adds them afterwards; this
+   * catches what slipped through. A cursor over binaries (as for single
+   * articles) visits each one once per pass, instead of searching all of
+   * them for parts on every batch (LOG.md 2026-10-04).
+   */
+  private copiedCursor = 0
   private async dropCopiedParts(budget: number): Promise<number> {
     let total = 0
     while (Date.now() < budget) {
-      const result = await db.rawQuery(
-        `DELETE FROM parts WHERE binary_id IN (
-           SELECT b.id FROM binaries b
-           WHERE b.file_id IS NOT NULL AND b.size IS NULL
-             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id)
-           LIMIT ?)`,
-        [500]
+      const { rows } = await db.rawQuery(
+        `WITH batch AS (
+           SELECT id FROM binaries WHERE id > ? AND file_id IS NOT NULL ORDER BY id LIMIT ?
+         ), gone AS (
+           DELETE FROM parts p USING batch WHERE p.binary_id = batch.id AND p.pcrc32 IS NULL
+           RETURNING 1)
+         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
+        [this.copiedCursor, COPIED_PARTS_BATCH]
       )
-      total += result.rowCount ?? 0
-      if (!result.rowCount) break
+      total += Number(rows[0].deleted)
+      if (rows[0].last === null) {
+        this.copiedCursor = 0
+        break
+      }
+      this.copiedCursor = Number(rows[0].last)
     }
     return total
   }
@@ -190,6 +308,30 @@ export default class GcScheduler extends BaseCommand {
     return total
   }
 
+  /**
+   * Completed header-derived binaries without a fingerprint, some hours after
+   * completion: their file exists, their parts are gone, and par2 naming needs
+   * a size and MD5 they don't have. Oldest first, via the last_seen_at index.
+   */
+  private async deleteUnfingerprintedBinaries(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.binaries.unfingerprintedAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM binaries WHERE id IN (
+           SELECT id FROM binaries
+           WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
+             AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
+             AND awaiting_inspection IS NOT TRUE
+           ORDER BY last_seen_at LIMIT ?)`,
+        [cutoff, gcConfig.deleteBatch]
+      )
+      total += result.rowCount
+      if (result.rowCount < gcConfig.deleteBatch) break
+    }
+    return total
+  }
+
   /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
   private async deleteEmptyReleases(budget: number): Promise<number> {
     const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)

commands/GroupAdd.ts

@@ -33,6 +33,9 @@ export default class GroupAdd extends BaseCommand {
   @flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' })
   declare reset: boolean
 
+  @flags.boolean({ description: "Start at the server's oldest article: the group's full history (also for groups indexed before)" })
+  declare fromStart: boolean
+
   public async run() {
     for (const name of this.names) {
       if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) {
@@ -42,26 +45,30 @@ export default class GroupAdd extends BaseCommand {
       const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies })
       group.active = !this.disable
       if (this.bodies) group.fetchBodies = true
-      if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
-        // The scheduler continues after last_queued_id.
-        const last = await this.lastArticle(name)
+      if (this.fromStart) {
+        // The scheduler continues after last_queued_id; 0 would mean "new group" to it.
+        const { first } = await this.groupInfo(name)
+        group.lastQueuedId = first > 1n ? first - 1n : 1n
+        group.lastIndexedId = group.lastQueuedId
+      } else if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
+        const { last } = await this.groupInfo(name)
         group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n)
         group.lastIndexedId = group.lastQueuedId
       }
       await group.save()
       this.logger.info(
         `${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` +
-          (this.backfill !== undefined ? `, starting after article ${group.lastQueuedId}` : '')
+          (this.backfill !== undefined || this.fromStart ? `, starting after article ${group.lastQueuedId}` : '')
       )
     }
-    if (this.backfill !== undefined) await NntpService.shutdown()
+    if (this.backfill !== undefined || this.fromStart) await NntpService.shutdown()
   }
 
-  private async lastArticle(name: string): Promise<bigint> {
+  private async groupInfo(name: string) {
     const conn = await NntpService.acquire()
     let failed = false
     try {
-      return parseGroupResponse((await conn.group(name)).resp).last
+      return parseGroupResponse((await conn.group(name)).resp)
     } catch (error) {
       failed = true
       throw error

commands/GroupBackfill.ts

@@ -0,0 +1,52 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import BackfillService from '#services/BackfillService'
+import { isSpotGroup } from '#services/SpotService'
+
+/**
+ * Indexes a group's older headers (CONCEPT.md D32): from where live
+ * indexing began back to N days ago, header-only and behind live indexing.
+ * index:scheduler queues the ranges; this only sets where to go.
+ */
+export default class GroupBackfill extends BaseCommand {
+  public static commandName = 'group:backfill'
+  public static description = "Indexes groups' older headers (header-only, behind live indexing)."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Groups (default: every active binary group)', required: false })
+  declare names?: string[]
+
+  @flags.number({ description: 'How many days back', default: 21 })
+  declare days: number
+
+  @flags.boolean({ description: 'Stop backfilling these groups (ranges already queued still run)' })
+  declare stop: boolean
+
+  public async run() {
+    const names = this.names?.length
+      ? this.names
+      : (await db.from('groups').where('active', true).orderBy('name').select('name')).map((g) => g.name as string).filter((n) => !isSpotGroup(n))
+    try {
+      for (const name of names) {
+        if (this.stop) {
+          await BackfillService.stop(name)
+          this.logger.info(`${name}: backfill stopped.`)
+          continue
+        }
+        const range = await BackfillService.start(name, this.days)
+        this.logger.info(
+          range
+            ? `${name}: backfilling articles ${range.from}–${range.to} (${(range.to - range.from).toLocaleString('en')} articles, ${this.days} days).`
+            : `${name}: nothing to backfill.`
+        )
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

commands/IndexScheduler.ts

@@ -5,8 +5,10 @@ import NntpService from '#services/NntpService'
 import QueueService from '#services/QueueService'
 import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
+import InspectionQueue from '#services/InspectionQueue'
 import FetchRangeService from '#services/FetchRangeService'
 import SplitService from '#services/SplitService'
+import BackfillService from '#services/BackfillService'
 
 export default class IndexScheduler extends BaseCommand {
   public static commandName = 'index:scheduler'
@@ -107,10 +109,33 @@ export default class IndexScheduler extends BaseCommand {
       }
     }
 
-    // Run immediately, then wait a full interval after each run so runs never overlap.
+    // The inspection backlog lives in Postgres; BullMQ gets a working set (LOG.md 2026-10-07 (8)).
+    const feedInspection = async () => {
+      const moved = await InspectionQueue.feed()
+      if (moved > 0) this.logger.info(`Inspection: ${moved} files moved from the backlog to the queue.`)
+    }
+
+    const backfill = async () => {
+      try {
+        const queued = await BackfillService.tick()
+        if (queued > 0) this.logger.info(`Queued ${queued} backfill range(s).`)
+      } catch (err: any) {
+        this.logger.error(`Error queuing backfill: ${err.message}`)
+      }
+    }
+
+    // Run immediately, then wait a full interval after each run so runs never overlap. A failed
+    // round (e.g. Postgres restarting) is logged and the next one tries again: an error that escaped
+    // here once ended the loop while the process stayed up, and live indexing stopped for ~3 hours
+    // (LOG.md 2026-10-07 (3)).
     while (!Shutdown.requested) {
-      await schedule()
-      await split()
+      for (const step of [schedule, backfill, split, feedInspection]) {
+        try {
+          await step()
+        } catch (err: any) {
+          this.logger.error(`Scheduler round failed, retrying next round: ${err.message}`)
+        }
+      }
       await Shutdown.sleep(indexerConfig.intervalMs)
     }
     await this.pool.shutdown()

commands/InspectionFromRedis.ts

@@ -0,0 +1,56 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import QueueService from '#services/QueueService'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
+
+/**
+ * One-off (LOG.md 2026-10-07 (8)): moves the inspection jobs waiting in
+ * BullMQ into the inspection_queue table, then empties BullMQ's waiting and
+ * prioritized jobs so Redis gives the memory back. The file id is read from
+ * the job id (file-<id>, content-<id>-<time>), so no job is loaded.
+ * index-scheduler then feeds a working set back in.
+ */
+export default class InspectionFromRedis extends BaseCommand {
+  public static commandName = 'inspection:from-redis'
+  public static description = 'Moves waiting inspection jobs from Redis into the inspection_queue table.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count, change nothing' })
+  declare dryRun: boolean
+
+  public async run() {
+    const queue = QueueService.collectionQueue
+    const client = await queue.client
+    const prefix = `${queue.opts.prefix ?? 'bull'}:${queue.name}`
+    try {
+      const idOf = (jobId: string) => {
+        const m = /^(?:file|content)-(\d+)/.exec(jobId)
+        return m ? Number(m[1]) : null
+      }
+      let moved = 0
+      let skipped = 0
+      const take = async (ids: string[], priority: number) => {
+        const fileIds = ids.map(idOf).filter((id): id is number => id !== null)
+        skipped += ids.length - fileIds.length
+        if (!this.dryRun) await InspectionQueue.add(fileIds, priority)
+        moved += fileIds.length
+      }
+      const waiting = await client.llen(`${prefix}:wait`)
+      for (let i = 0; i < waiting; i += 50_000) await take(await client.lrange(`${prefix}:wait`, i, i + 49_999), INSPECTION_PRIORITY.normal)
+      // Prioritized jobs were the low-priority niconico captures.
+      const prioritized = await client.zcard(`${prefix}:prioritized`)
+      for (let i = 0; i < prioritized; i += 50_000) await take(await client.zrange(`${prefix}:prioritized`, i, i + 49_999), INSPECTION_PRIORITY.low)
+      this.logger.info(`${this.dryRun ? 'Would move' : 'Moved'} ${moved} jobs into inspection_queue (${skipped} with other ids skipped).`)
+      if (this.dryRun) return
+      // Waiting and prioritized jobs only; active ones finish as they are.
+      await queue.drain()
+      const counts = await queue.getJobCounts('wait', 'prioritized', 'active')
+      this.logger.info(`BullMQ now: ${counts.wait} waiting, ${counts.prioritized} prioritized, ${counts.active} active; table: ${await InspectionQueue.size()}.`)
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+}

commands/ReleaseMergeSplit.ts

@@ -0,0 +1,21 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+
+/** Merges uploads split per poster (gc:scheduler also does this every sweep). */
+export default class ReleaseMergeSplit extends BaseCommand {
+  public static commandName = 'release:merge-split'
+  public static description = 'Merges releases of one upload that was posted under a different poster per file.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const { names, releases } = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    this.logger.info(`${names} split uploads; ${releases} releases ${this.dryRun ? 'would be' : ''} merged into their first.`)
+  }
+}

commands/ReleaseRebuild.ts

@@ -70,7 +70,7 @@ export default class ReleaseRebuild extends BaseCommand {
                AND NOT (v.release_id IS NULL
                         AND b.release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))
                -- A posted NZB decided this grouping (NzbImportService); name rules don't override it.
-               AND NOT EXISTS (SELECT 1 FROM files n WHERE n.release_id = b.release_id AND n.content_type = 'nzb')`,
+               AND NOT EXISTS (SELECT 1 FROM files n JOIN file_details nd ON nd.file_id = n.id WHERE n.release_id = b.release_id AND nd.content_type = 'nzb')`,
             [ids, releaseIdsPerBinary] as any
           )
         })
@@ -85,12 +85,13 @@ export default class ReleaseRebuild extends BaseCommand {
         const rows = await db
           .from('files as f')
           .join('binaries as b', 'b.file_id', 'f.id')
-          .whereNotNull('f.archive')
+          .join('file_details as d', 'd.file_id', 'f.id')
+          .whereNotNull('d.archive')
           .whereNull('b.release_id')
           .where('f.id', '>', rarCursor)
           .orderBy('f.id')
           .limit(BATCH)
-          .select('f.id as file_id', 'f.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
+          .select('f.id as file_id', 'd.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
         if (rows.length === 0) break
         rarCursor = Number(rows.at(-1).file_id)
         const segments = rows.map((r) => ({
@@ -147,7 +148,8 @@ export default class ReleaseRebuild extends BaseCommand {
     await db.transaction(async (trx) => {
       await trx.rawQuery('DELETE FROM release_files')
       await trx.rawQuery(`UPDATE binaries SET real_name = NULL WHERE real_name IS NOT NULL`)
-      await trx.rawQuery(`UPDATE files SET real_name = NULL, archive = NULL WHERE real_name IS NOT NULL OR archive IS NOT NULL`)
+      await trx.rawQuery(`UPDATE files SET real_name = NULL WHERE real_name IS NOT NULL`)
+      await trx.rawQuery(`UPDATE file_details SET archive = NULL WHERE archive IS NOT NULL`)
       await trx.rawQuery(`UPDATE releases SET contents = NULL, encrypted = NULL WHERE contents IS NOT NULL OR encrypted IS NOT NULL`)
       // Binaries whose name rules find no release were grouped from contents (or wrongly by size).
       await trx.rawQuery(`

commands/ReleaseRename.ts

@@ -1,6 +1,6 @@
-import { BaseCommand } from '@adonisjs/core/ace'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
-import { renameFromStoredContents } from '#services/ArchiveService'
+import { renameFromStoredContents, renameHiddenNamed } from '#services/ArchiveService'
 import { regroupAndRenameFromPar2 } from '#services/NameService'
 
 /** Renames random-named releases from their stored archive listings (no downloads). */
@@ -12,7 +12,14 @@ export default class ReleaseRename extends BaseCommand {
     startApp: true,
   }
 
+  @flags.boolean({ description: 'Only releases named after a hidden archive entry (e.g. a decoy ".x.mkv")' })
+  declare hidden: boolean
+
   public async run() {
+    if (this.hidden) {
+      this.logger.info(`Renamed ${await renameHiddenNamed()} releases named after a hidden archive entry.`)
+      return
+    }
     const par2 = await regroupAndRenameFromPar2()
     this.logger.info(`par2: ${par2.regrouped} files moved into the set that names them, ${par2.renamed} releases renamed.`)
     this.logger.info(`Renamed ${await renameFromStoredContents()} releases from their archive contents.`)

commands/SeedNode.ts

@@ -0,0 +1,461 @@
+import { gzipSync, unzipSync } from 'node:zlib'
+import { spawn, type ChildProcess } from 'node:child_process'
+import { availableParallelism } from 'node:os'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import nntpConfig from '#config/nntp'
+import indexerConfig from '#config/indexer'
+import { XoverClient, parseOverviews, type Overview } from '#lib/nntp/xover_client'
+import { runRange, type OverviewSource } from '#lib/seed/runner'
+import type { SeedBatch } from '#lib/seed/wire'
+import type { OverviewHeader } from '#lib/ingest/segment'
+import { probeInspection, probeWalk, type ContentTask } from '#lib/content/probe'
+
+class LeaseLost extends Error {}
+class Stopping extends Error {}
+
+interface InspectJob {
+  queue: string
+  jobId: string
+  token: string
+  task: ContentTask
+}
+
+interface Lease {
+  id: number
+  group: string
+  start: number
+  end: number
+}
+
+/**
+ * A seeding node (CONCEPT.md D38): leases article ranges from the seed API
+ * over HTTPS, fetches and aggregates their headers over its own NNTP
+ * connection(s) (D37), and posts the results back in gzip-compressed
+ * batches. Outgoing requests only: no ports, database or Redis needed.
+ *
+ *   SEED_URL=https://seed.example SEED_TOKEN=sn_… node ace seed:node --ranges 4 --inspectors 2
+ */
+export default class SeedNode extends BaseCommand {
+  public static commandName = 'seed:node'
+  public static description =
+    'Runs a seeding node: leases ranges from the seed API and submits what it aggregated.'
+
+  public static options: CommandOptions = {
+    startApp: false,
+    staysAlive: true,
+  }
+
+  @flags.string({ description: 'Seed API base URL (default: SEED_URL)' })
+  declare url?: string
+
+  @flags.string({ description: 'Node token (default: SEED_TOKEN)' })
+  declare token?: string
+
+  @flags.number({
+    description:
+      'Ranges seeded at once (with SEED_FETCH=direct, one NNTP connection each); 0 for an inspection-only node (default: SEED_RANGES or 1)',
+    default: process.env.SEED_RANGES ? Number(process.env.SEED_RANGES) : 1,
+  })
+  declare ranges: number
+
+  @flags.number({
+    description:
+      'Content inspections and archive walks worked on at once, articles through the seed API (default: SEED_INSPECTORS or 0)',
+    default: Number(process.env.SEED_INSPECTORS) || 0,
+  })
+  declare inspectors: number
+
+  @flags.number({ description: 'Articles per XOVER', default: 20_000 })
+  declare chunk: number
+
+  @flags.number({ description: 'XOVER chunks per batch posted to the API', default: 5 })
+  declare submitEvery: number
+
+  @flags.string({
+    description:
+      'relay: headers come through the seed API (no NNTP account needed); direct: own NNTP connections (default: SEED_FETCH or relay)',
+    default: process.env.SEED_FETCH || 'relay',
+  })
+  declare fetch: string
+
+  @flags.number({
+    description:
+      'Processes the ranges and inspections are spread over; one process uses one core (default: SEED_PROCESSES or the number of cores)',
+    default: process.env.SEED_PROCESSES ? Number(process.env.SEED_PROCESSES) : availableParallelism(),
+  })
+  declare processes: number
+
+  @flags.boolean({ description: 'Stop when there is nothing to lease instead of waiting' })
+  declare once: boolean
+
+  private stopping = false
+  /** In a worker process: its number, in front of its log lines. */
+  private readonly child = process.env.SEED_PROCESS ?? ''
+
+  public async run() {
+    const base = (this.url ?? process.env.SEED_URL ?? '').replace(/\/+$/, '')
+    const token = this.token ?? process.env.SEED_TOKEN ?? ''
+    if (!base || !token) {
+      this.logger.error('Set --url and --token (or SEED_URL and SEED_TOKEN).')
+      this.exitCode = 1
+      return
+    }
+    // JavaScript runs on one core per process: several slots are spread over worker processes.
+    const processes = Math.max(1, Math.min(this.processes || 1, this.ranges + this.inspectors))
+    if (!this.child && processes > 1) return this.supervise(base, token, processes)
+
+    const stop = () => {
+      if (!this.stopping && !this.child) this.logger.info('Stopping: giving back the ranges in progress…')
+      this.stopping = true
+    }
+    process.on('SIGTERM', stop)
+    process.on('SIGINT', stop)
+
+    const api = new SeedApi(base, token, (message) => this.logger.warning(message))
+    if (!this.child) this.logger.info(
+      `Seeding node for ${base}: ${this.ranges} range(s) at once, ${this.fetch === 'direct' ? 'own NNTP connections' : 'headers through the seed API'}` +
+        (this.inspectors > 0 ? `; ${this.inspectors} inspection(s) at once, articles through the seed API.` : '.')
+    )
+    await Promise.all([
+      ...Array.from({ length: this.ranges }, (_, i) => this.work(api, i + 1)),
+      ...Array.from({ length: this.inspectors }, (_, i) => this.inspect(api, i + 1)),
+    ])
+    if (!this.child) this.logger.info('Seeding node stopped.')
+  }
+
+  /**
+   * Runs the slots in `processes` worker processes (this command again, with
+   * SEED_PROCESS set), each with its share of the ranges and inspections. A
+   * worker that exits while the node runs is restarted; SIGTERM/SIGINT are
+   * passed on, and the node stops when all workers have given their work back.
+   */
+  private async supervise(base: string, token: string, processes: number) {
+    // Slots dealt out in turn (ranges first, then inspections), so every process has work.
+    const share = (from: number, to: number, i: number) =>
+      Array.from({ length: to - from }, (_, k) => from + k).filter((k) => k % processes === i).length
+    this.logger.info(
+      `Seeding node for ${base}: ${this.ranges} range(s) and ${this.inspectors} inspection(s) at once, over ${processes} processes, ` +
+        `${this.fetch === 'direct' ? 'own NNTP connections' : 'articles and headers through the seed API'}.`
+    )
+    const children = new Set<ChildProcess>()
+    const start = (n: number): Promise<void> =>
+      new Promise((resolve) => {
+        const args = [
+          ...process.execArgv,
+          process.argv[1],
+          'seed:node',
+          `--ranges=${share(0, this.ranges, n - 1)}`,
+          `--inspectors=${share(this.ranges, this.ranges + this.inspectors, n - 1)}`,
+          `--fetch=${this.fetch}`,
+          `--chunk=${this.chunk}`,
+          `--submit-every=${this.submitEvery}`,
+          ...(this.once ? ['--once'] : []),
+        ]
+        const child = spawn(process.execPath, args, {
+          env: { ...process.env, SEED_URL: base, SEED_TOKEN: token, SEED_PROCESS: String(n) },
+          stdio: 'inherit',
+        })
+        children.add(child)
+        child.on('exit', (code, signal) => {
+          children.delete(child)
+          if (this.stopping || this.once || code === 0) return resolve()
+          this.logger.warning(`Process ${n} exited (${signal ?? code}); restarting in 10 s`)
+          setTimeout(() => start(n).then(resolve), 10_000)
+        })
+      })
+    const stop = (signal: NodeJS.Signals) => {
+      if (!this.stopping) this.logger.info('Stopping: giving back the ranges and inspections in progress…')
+      this.stopping = true
+      for (const child of children) child.kill(signal)
+    }
+    process.on('SIGTERM', () => stop('SIGTERM'))
+    process.on('SIGINT', () => stop('SIGINT'))
+    await Promise.all(Array.from({ length: processes }, (_, i) => start(i + 1)))
+    this.logger.info('Seeding node stopped.')
+  }
+
+  private async work(api: SeedApi, worker: number) {
+    const log = (message: string) => this.logger.info(`[${this.child && `${this.child}/`}${worker}] ${message}`)
+    // The NNTP connection is opened for a lease and kept while leases keep coming; with
+    // nothing to lease it's closed, so an idle node holds no connections.
+    let client: XoverClient | undefined
+    try {
+      while (!this.stopping) {
+        const lease = await this.lease(api, log)
+        if (!lease) {
+          client?.close()
+          client = undefined
+          if (this.once) return
+          await this.pause(60_000)
+          continue
+        }
+        if (this.fetch !== 'direct') {
+          await this.workRange(api, new RelaySource(api, lease.id), lease, log)
+          continue
+        }
+        if (!client) {
+          try {
+            client = await this.connect()
+          } catch (error: any) {
+            // Never started: the lease doesn't count as one of the range's attempts.
+            const ipLimit = /simultaneous IP/i.test(error.message)
+            log(`NNTP login failed: ${error.message}; trying again in ${ipLimit ? 5 : 1} min`)
+            await api
+              .post(`/api/v1/seed/ranges/${lease.id}/release`, {
+                error: `NNTP login: ${error.message}`,
+                uncounted: true,
+              })
+              .catch(() => {})
+            await this.pause(ipLimit ? 300_000 : 60_000)
+            continue
+          }
+        }
+        if (!(await this.workRange(api, client, lease, log))) {
+          client.close()
+          client = undefined
+        }
+      }
+    } finally {
+      client?.close()
+    }
+  }
+
+  /**
+   * Content inspection and archive walks (CONCEPT.md D41): lease a few jobs,
+   * probe each with articles through the seed API, post the results. The
+   * indexer applies them; a job left unanswered goes back after 10 minutes.
+   */
+  private async inspect(api: SeedApi, slot: number) {
+    const log = (message: string) => this.logger.info(`[inspect ${this.child && `${this.child}/`}${slot}] ${message}`)
+    const fetchArticle = (id: string) => api.get(`/api/v1/seed/articles?id=${encodeURIComponent(id)}`, 4)
+    while (!this.stopping) {
+      let jobs: InspectJob[] = []
+      try {
+        jobs = ((await api.post('/api/v1/seed/inspect/lease', { max: 2 })) as { jobs: InspectJob[] } | null)?.jobs ?? []
+      } catch (error: any) {
+        if (error instanceof Unauthorized) throw error
+        log(`Lease failed: ${error.message}`)
+      }
+      if (jobs.length === 0) {
+        if (this.once) return
+        await this.pause(30_000)
+        continue
+      }
+      for (const job of jobs) {
+        const ref = { queue: job.queue, jobId: job.jobId, token: job.token }
+        if (this.stopping) {
+          await api.post('/api/v1/seed/inspect/release', ref).catch(() => {})
+          continue
+        }
+        try {
+          const result =
+            job.task.kind === 'inspect'
+              ? await probeInspection(job.task, fetchArticle, (message) => log(message))
+              : await probeWalk(job.task, fetchArticle)
+          await api.post('/api/v1/seed/inspect/done', { ...ref, result: JSON.stringify(result) })
+        } catch (error: any) {
+          if (error instanceof Unauthorized) throw error
+          // The job's lock expired and it went to another node: nothing to report.
+          if (error instanceof LeaseLost) continue
+          log(`${job.task.kind === 'inspect' ? `"${job.task.name}"` : `walk of release ${job.task.releaseId}`} failed: ${error.message}`)
+          await api.post('/api/v1/seed/inspect/failed', { ...ref, error: error.message }).catch(() => {})
+        }
+      }
+    }
+  }
+
+  private connect() {
+    return XoverClient.connect({
+      host: nntpConfig.host,
+      port: nntpConfig.port,
+      secure: nntpConfig.secure,
+      servername: nntpConfig.tlsServername,
+      user: nntpConfig.user,
+      password: nntpConfig.password?.release(),
+      timeoutMs: nntpConfig.commandTimeoutMs,
+      compress: true,
+    })
+  }
+
+  /** The next range, or null when there is none (or the API can't be reached; logged). */
+  private async lease(api: SeedApi, log: (message: string) => void): Promise<Lease | null> {
+    try {
+      return ((await api.post('/api/v1/seed/lease')) as { range: Lease } | null)?.range ?? null
+    } catch (error: any) {
+      if (error instanceof Unauthorized) throw error
+      log(`Lease failed: ${error.message}`)
+      return null
+    }
+  }
+
+  /** Works one leased range; false when the NNTP connection should be replaced. */
+  private async workRange(
+    api: SeedApi,
+    client: OverviewSource,
+    lease: Lease,
+    log: (message: string) => void
+  ): Promise<boolean> {
+    log(`${lease.group} ${lease.start}–${lease.end}`)
+    const started = Date.now()
+    let headers = 0
+    let chain: Promise<void> = Promise.resolve()
+    // Where the range resumes after what was stored: a batch split in two reports it with its first half.
+    let reported = lease.start
+    const posting = new Set<Promise<void>>()
+    // Throws once a post has failed, so the range stops fetching.
+    const failedSoFar = () => Promise.race([chain, Promise.resolve()])
+    try {
+      await runRange({
+        client,
+        group: lease.group,
+        start: lease.start,
+        end: lease.end,
+        chunk: this.chunk,
+        submitEvery: this.submitEvery,
+        isDeferred: (header: OverviewHeader) =>
+          indexerConfig.deferBodyMessageIds.some((re) => re.test(header['message-id'] ?? '')),
+        // Batches are posted in order, one after the other, while fetching goes on; with two
+        // waiting, fetching waits too. A failed post stops the chain, so no batch is stored
+        // past a missing one (the range would resume beyond it).
+        sink: async (batch: SeedBatch) => {
+          if (this.stopping) throw new Stopping()
+          while (posting.size >= 2) await Promise.race(posting)
+          const post = chain.then(async () => {
+            await this.postBatch(api, lease.id, batch, reported)
+            reported = batch.next
+            headers += batch.headers
+          })
+          chain = post
+          posting.add(post)
+          post.catch(() => {}).finally(() => posting.delete(post))
+          await failedSoFar()
+        },
+      })
+      await chain
+      await api.post(`/api/v1/seed/ranges/${lease.id}/done`)
+      const seconds = (Date.now() - started) / 1000
+      log(
+        `done: ${headers.toLocaleString('en')} headers in ${seconds.toFixed(0)} s (${Math.round(headers / seconds).toLocaleString('en')}/s)`
+      )
+      return true
+    } catch (error: any) {
+      if (error instanceof Unauthorized) throw error
+      // An XOVER may still be in flight: the connection is replaced either way.
+      if (error instanceof LeaseLost) {
+        log(`Range ${lease.id} passed to another node; dropping it.`)
+        return false
+      }
+      const reason = error instanceof Stopping ? 'node stopping' : error.message
+      if (!(error instanceof Stopping)) log(`Range ${lease.id} failed: ${reason}`)
+      // Let posts in flight finish (or fail) first, so the range resumes after what was stored.
+      await chain.catch(() => {})
+      await api.post(`/api/v1/seed/ranges/${lease.id}/release`, { error: reason }).catch(() => {})
+      if (!this.stopping) await this.pause(30_000)
+      return false
+    }
+  }
+
+  /**
+   * Posts a batch; one the seed API finds too large (413: dense recent ranges) is
+   * posted in two halves, at most three splits deep. Only the second half moves
+   * the range's resume point, so a failure in between loses nothing.
+   */
+  private async postBatch(api: SeedApi, rangeId: number, batch: SeedBatch, reported: number, depth = 0): Promise<void> {
+    try {
+      await api.post(`/api/v1/seed/ranges/${rangeId}/batches`, batch)
+    } catch (error: any) {
+      const parts = batch.files.length + batch.fragments.length
+      if (!/: 413 /.test(String(error.message)) || depth >= 3 || parts < 2) throw error
+      const files = Math.ceil(batch.files.length / 2)
+      const fragments = Math.ceil(batch.fragments.length / 2)
+      const first: SeedBatch = { ...batch, next: reported, headers: 0, obfuscated: 0, deferred: 0, files: batch.files.slice(0, files), fragments: batch.fragments.slice(0, fragments) }
+      const second: SeedBatch = { ...batch, files: batch.files.slice(files), fragments: batch.fragments.slice(fragments) }
+      await this.postBatch(api, rangeId, first, reported, depth + 1)
+      await this.postBatch(api, rangeId, second, reported, depth + 1)
+    }
+  }
+
+  /** Sleeps, but wakes up early when the node is stopping. */
+  private async pause(ms: number) {
+    for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)
+  }
+}
+
+class Unauthorized extends Error {}
+
+/** The seed API client: JSON over HTTPS, gzip request bodies, retries for network errors and 5xx. */
+class SeedApi {
+  constructor(
+    private base: string,
+    private token: string,
+    private warn: (message: string) => void
+  ) {}
+
+  async post(path: string, body?: unknown): Promise<unknown> {
+    const payload = body === undefined ? undefined : gzipSync(JSON.stringify(body))
+    const result = await this.request('POST', path, payload)
+    if (result === null) return null
+    const text = result.toString('utf8')
+    return text ? JSON.parse(text) : null
+  }
+
+  /** A binary GET (a relay chunk, an article). */
+  async get(path: string, attempts = 14): Promise<Buffer> {
+    return (await this.request('GET', path, undefined, attempts)) ?? Buffer.alloc(0)
+  }
+
+  private async request(method: string, path: string, payload?: Buffer, attempts = 14): Promise<Buffer | null> {
+    for (let attempt = 1; ; attempt++) {
+      let status = 0
+      let data = Buffer.alloc(0)
+      try {
+        const response = await fetch(this.base + path, {
+          method,
+          headers: {
+            authorization: `Bearer ${this.token}`,
+            ...(payload ? { 'content-type': 'application/json', 'content-encoding': 'gzip' } : {}),
+          },
+          body: payload,
+          signal: AbortSignal.timeout(10 * 60 * 1000),
+        })
+        status = response.status
+        data = Buffer.from(await response.arrayBuffer())
+      } catch (error: any) {
+        data = Buffer.from(error.message)
+      }
+      const text = () => data.toString('utf8').slice(0, 300)
+      if (status === 204) return null
+      if (status >= 200 && status < 300) return data
+      if (status === 401) throw new Unauthorized('The seed API rejected this node token')
+      if (status === 409) throw new LeaseLost(text())
+      // 404 comes from the proxy while the seed API restarts (a deploy): this API has no 404s of its own.
+      if (status >= 400 && status < 500 && status !== 404)
+        throw new Error(`Seed API ${path}: ${status} ${text()}`)
+      // Network errors, 404 and 5xx: storing a batch twice is harmless, so try again.
+      // ~20 minutes in all, longer than a lease lasts: a restart or maintenance of the API is waited out.
+      if (attempt >= attempts) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text()}`)
+      const wait = Math.min(120_000, 2000 * 2 ** (attempt - 1))
+      this.warn(`Seed API ${path}: ${status || text()}; retrying in ${wait / 1000} s`)
+      await sleep(wait)
+    }
+  }
+}
+
+/** A range's overviews through the seed API's relay (CONCEPT.md D38): no news server account on the node. */
+class RelaySource implements OverviewSource {
+  constructor(
+    private api: SeedApi,
+    private rangeId: number
+  ) {}
+
+  async group() {}
+
+  async xover(from: number, to: number): Promise<Overview[]> {
+    const raw = await this.api.get(
+      `/api/v1/seed/ranges/${this.rangeId}/chunk?from=${from}&to=${to}`
+    )
+    return raw.length > 0 ? parseOverviews(unzipSync(raw).toString('utf8')) : []
+  }
+}

commands/SeedPlan.ts

@@ -0,0 +1,47 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import NntpService from '#services/NntpService'
+import SeedOrchestrator from '#services/SeedOrchestrator'
+
+/**
+ * Plans seed ranges (CONCEPT.md D38) for groups: every article from below
+ * what live indexing and the backfill cover down to `--days` back (or the
+ * server's oldest article) that no earlier range covers. Running it again
+ * plans only what's missing. A group's backfill stops where its plan
+ * starts. Ranges are interleaved across groups, newest first.
+ *
+ *   node ace seed:plan alt.binaries.ath alt.binaries.mom --days 365
+ */
+export default class SeedPlan extends BaseCommand {
+  public static commandName = 'seed:plan'
+  public static description = 'Plans article ranges of groups for seeding nodes to lease.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Groups' })
+  declare groups: string[]
+
+  @flags.number({ description: 'How many days back (default: everything the server has)' })
+  declare days?: number
+
+  @flags.number({ description: 'Articles per range', default: 2_000_000 })
+  declare size: number
+
+  @flags.number({ description: 'Leased before ranges of a lower priority (default 0)', default: 0 })
+  declare priority: number
+
+  public async run() {
+    try {
+      const summary = await SeedOrchestrator.plan(this.groups, { days: this.days, size: this.size, priority: this.priority })
+      for (const s of summary) {
+        this.logger.info(
+          `${s.group}: ${s.ranges.toLocaleString('en')} new ranges in ${s.gaps} gap(s) not yet planned, articles ${s.from.toLocaleString('en')}–${s.to.toLocaleString('en')}.`
+        )
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

commands/SeedRange.ts

@@ -0,0 +1,126 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import nntpConfig from '#config/nntp'
+import SeedService from '#services/SeedService'
+import QueueService from '#services/QueueService'
+import { isDeferred } from '#services/DeferredService'
+import { XoverClient } from '#lib/nntp/xover_client'
+import { runRange } from '#lib/seed/runner'
+import type { SeedBatch } from '#lib/seed/wire'
+
+/**
+ * Edge aggregation (CONCEPT.md D37) on this machine: indexes a range of a
+ * group's history over one NNTP connection (gzip-compressed overviews when
+ * the server has them), grouping headers into files in memory and storing
+ * one row per file directly. seed:node does the same for a remote node,
+ * through the seed API. Header-only, like the backfill.
+ *
+ *   node ace seed:range alt.binaries.ath --start 46000000000 --count 2000000
+ *   node ace seed:range alt.binaries.ath --start 46000000000 --count 2000000 --dry-run
+ */
+export default class SeedRange extends BaseCommand {
+  public static commandName = 'seed:range'
+  public static description = "Indexes a range of a group's headers by aggregating them into files on this node."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.string({ description: 'Group' })
+  declare group: string
+
+  @flags.string({ description: 'First article number' })
+  declare start: string
+
+  @flags.number({ description: 'Articles to index', default: 1_000_000 })
+  declare count: number
+
+  @flags.number({ description: 'Articles per XOVER', default: 20_000 })
+  declare chunk: number
+
+  @flags.number({ description: 'Batches without a new part before an open file is given up as fragments', default: 5 })
+  declare idle: number
+
+  @flags.boolean({ description: 'Fetch and aggregate only; store nothing' })
+  declare dryRun: boolean
+
+  @flags.boolean({ description: 'Ask the server for gzip-compressed overviews', default: true })
+  declare compress: boolean
+
+  public async run() {
+    const first = Number(this.start)
+    if (!Number.isSafeInteger(first) || first < 1) throw new Error(`--start must be an article number, got ${this.start}`)
+    const last = first + this.count - 1
+    const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, dropped: 0, fragments: 0, inspections: 0 }
+    let storeMs = 0
+    const cpuStart = process.cpuUsage()
+    const started = Date.now()
+
+    const client = await XoverClient.connect({
+      host: nntpConfig.host,
+      port: nntpConfig.port,
+      secure: nntpConfig.secure,
+      servername: nntpConfig.tlsServername,
+      user: nntpConfig.user,
+      password: nntpConfig.password?.release(),
+      timeoutMs: nntpConfig.commandTimeoutMs,
+      compress: this.compress,
+    })
+    this.logger.info(`Connected${client.compressed ? ', overviews gzip-compressed' : ''}.`)
+    try {
+      await runRange({
+        client,
+        group: this.group,
+        start: first,
+        end: last,
+        chunk: this.chunk,
+        idle: this.idle,
+        submitEvery: 1,
+        isDeferred,
+        sink: async (batch: SeedBatch) => {
+          stats.headers += batch.headers
+          stats.obfuscated += batch.obfuscated
+          stats.deferred += batch.deferred
+          stats.complete += batch.files.length
+          stats.fragments += batch.fragments.length
+          if (!this.dryRun) {
+            const t = Date.now()
+            const stored = await SeedService.store(batch.files)
+            await SeedService.storeFragments(batch.fragments, { start: first, end: last })
+            storeMs += Date.now() - t
+            stats.files += stored.files
+            stats.existing += stored.existing
+            stats.dropped += stored.dropped
+            stats.inspections += stored.inspections
+          }
+          this.progress(stats, started)
+        },
+      })
+    } finally {
+      client.close()
+      await QueueService.closeAll()
+    }
+
+    const seconds = (Date.now() - started) / 1000
+    const cpu = process.cpuUsage(cpuStart)
+    const n = (x: number) => x.toLocaleString('en')
+    this.logger.info(
+      `${this.group} ${first}-${last}: ${n(stats.headers)} headers in ${seconds.toFixed(0)} s (${n(Math.round(stats.headers / seconds))}/s). ` +
+        `${n(stats.obfuscated)} obfuscated, ${n(stats.deferred)} deferred. ` +
+        `${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known, ${n(stats.dropped)} garbage), ` +
+        `${n(stats.fragments)} fragment segments, ${n(stats.inspections)} queued for inspection.`
+    )
+    this.logger.info(`Store ${(storeMs / 1000).toFixed(0)} s; node CPU ${((cpu.user + cpu.system) / 1e6).toFixed(0)} s.`)
+  }
+
+  private lastReport = 0
+  private progress(stats: Record<string, number>, started: number) {
+    if (Date.now() - this.lastReport < 30_000) return
+    this.lastReport = Date.now()
+    const seconds = (Date.now() - started) / 1000
+    this.logger.info(
+      `${stats.headers.toLocaleString('en')} headers (${Math.round(stats.headers / seconds).toLocaleString('en')}/s), ` +
+        `${stats.complete.toLocaleString('en')} files, ${stats.fragments.toLocaleString('en')} fragment segments.`
+    )
+  }
+}

commands/SeedStatus.ts

@@ -0,0 +1,26 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import SeedOrchestrator from '#services/SeedOrchestrator'
+
+/** Seeding progress per range status and per node (CONCEPT.md D38). */
+export default class SeedStatus extends BaseCommand {
+  public static commandName = 'seed:status'
+  public static description = 'Shows seeding progress per range status and per node.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const status = await SeedOrchestrator.status()
+    const n = (x: unknown) => Number(x ?? 0).toLocaleString('en')
+    for (const r of status.ranges) {
+      this.logger.info(`${r.status}: ${n(r.ranges)} ranges, ${n(r.reached)} of ${n(r.articles)} articles reached, ${n(r.headers)} headers, ${n(r.files)} files`)
+    }
+    for (const node of status.nodes) {
+      this.logger.info(`node ${node.name}${node.disabled ? ' (disabled)' : ''}: ${node.leases} leases, ${n(node.headers)} headers, last seen ${node.last_seen_at ?? 'never'}`)
+    }
+    this.logger.info(`Headers in ranges finished in the last hour: ${n(status.headersDoneLastHour)}`)
+    if (status.paused) this.logger.warning(`No new leases: ${status.paused} (BACKFILL_MAX_NVME_GB / BACKFILL_MAX_HDD_GB).`)
+  }
+}

commands/SeedToken.ts

@@ -0,0 +1,37 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import SeedOrchestrator from '#services/SeedOrchestrator'
+
+/**
+ * Creates a seeding node, or gives it a new token (the old one stops
+ * working), and prints the token once (CONCEPT.md D38).
+ *
+ *   node ace seed:token hetzner-1
+ *   node ace seed:token hetzner-1 --disable
+ */
+export default class SeedToken extends BaseCommand {
+  public static commandName = 'seed:token'
+  public static description = 'Creates a seeding node or replaces its token; prints the token once.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.string({ description: 'Node name' })
+  declare name: string
+
+  @flags.boolean({ description: 'Disable the node instead (its leases pass to others after they expire)' })
+  declare disable: boolean
+
+  public async run() {
+    if (this.disable) {
+      const n = await db.from('seed_nodes').where('name', this.name).update({ disabled: true })
+      this.logger.info(n ? `${this.name}: disabled.` : `${this.name}: no such node.`)
+      return
+    }
+    const token = await SeedOrchestrator.issueToken(this.name)
+    this.logger.info(`${this.name}: token (shown once, keep it on the node only):`)
+    console.log(token)
+  }
+}

commands/SpotPasswords.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import SpotService from '#services/SpotService'
+
+/** Passwords from the titles and descriptions of spots read so far, onto their releases. */
+export default class SpotPasswords extends BaseCommand {
+  public static commandName = 'spot:passwords'
+  public static description = "Takes archive passwords from spots' titles and descriptions onto their releases."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const spots = await db
+      .from('spots')
+      .whereNotNull('release_id')
+      .where((q) => q.whereNotNull('description').orWhere('title', 'like', '%{{%}}%'))
+      .select('message_id')
+    let found = 0
+    for (const spot of spots) if (await SpotService.applyPassword(spot.message_id)) found++
+    this.logger.info(`${found} of ${spots.length} spots name a password in their title or description.`)
+  }
+}

commands/SpotPostprocess.ts

@@ -0,0 +1,55 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+const BATCH = 500
+
+/**
+ * Reads the article and NZB of listed spots that were listed from their
+ * overview only (LOG.md 2026-10-08 (10)): description, the NZB's password or
+ * one in the spot's text, placeholder NZBs. Newest first; can be stopped and
+ * started again (worker:spots does the same when idle, a batch at a time).
+ * Connections: NNTP_POOL_SIZE of this process, --concurrency at a time.
+ */
+export default class SpotPostprocess extends BaseCommand {
+  public static commandName = 'spot:postprocess'
+  public static description = "Reads listed spots' articles and NZBs for their passwords and details."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Spots read at a time (default: NNTP_POOL_SIZE)' })
+  declare concurrency?: number
+
+  @flags.number({ description: 'Stop after this many spots' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = this.concurrency ?? env.get('NNTP_POOL_SIZE', 4)
+    const [{ count }] = await db.from('spots').where('status', 'listed').whereNull('nzb_read_at').count('* as count')
+    this.logger.info(`${count} listed spots to read, ${concurrency} at a time.`)
+    const totals: Record<string, number> = {}
+    const started = Date.now()
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const counts = await SpotService.readListings(Math.min(BATCH, this.limit ? this.limit - done : BATCH), concurrency)
+        const read = Object.values(counts).reduce((sum, n) => sum + n, 0)
+        if (read === 0) break
+        for (const [k, v] of Object.entries(counts)) totals[k] = (totals[k] ?? 0) + v
+        done += read
+        const rate = done / ((Date.now() - started) / 1000)
+        this.logger.info(`${done} read (${rate.toFixed(1)}/s): ${Object.entries(totals).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+        // Usenet unreachable: everything was given back; don't spin on it.
+        if (counts.retry === read) await Shutdown.sleep(30_000)
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

commands/SpotReimport.ts

@@ -0,0 +1,51 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+/**
+ * Imported spots whose files sit in a release that isn't the spot's own
+ * (another source, or another name): their NZB is read again, which moves
+ * the files into the spot's release under its title (CONCEPT.md D29).
+ */
+export default class SpotReimport extends BaseCommand {
+  public static commandName = 'spot:reimport'
+  public static description = "Moves imported spots' files into their own release, named after the spot."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const spots = await db
+      .from('spots as s')
+      .join('releases as r', 'r.id', 's.release_id')
+      .where('s.status', 'imported')
+      .where((q) => q.whereNot('r.source', 'spot').orWhereRaw('r.name <> s.title'))
+      .orderBy('s.posted_at', 'desc')
+      .select('s.*', 'r.name as release_name')
+    this.logger.info(`${spots.length} imported spots are in a release that isn't their own.`)
+    if (this.dryRun) return
+    let moved = 0
+    let failed = 0
+    try {
+      for (const spot of spots) {
+        try {
+          const releaseId = await SpotService.reimport(spot)
+          if (releaseId) moved++
+          else failed++
+        } catch (error: any) {
+          failed++
+          this.logger.warning(`${spot.title}: ${error.message}`)
+        }
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+    this.logger.info(`${moved} spots re-imported, ${failed} without a usable NZB (dummy, gone) or failed.`)
+  }
+}

commands/workers/Collection.ts

@@ -6,7 +6,8 @@ import env from '#start/env'
 import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
 import NntpService from '#services/NntpService'
-import ContentService from '#services/ContentService'
+import ContentService, { describeContent } from '#services/ContentService'
+import ArchiveWalkService from '#services/ArchiveWalkService'
 import File from '#models/file'
 
 export default class CollectionWorker extends BaseCommand {
@@ -32,27 +33,7 @@ export default class CollectionWorker extends BaseCommand {
 
       try {
         const content = await ContentService.inspect(fileId)
-        if (content?.par2) {
-          const { par2 } = content
-          this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
-        } else if (content?.archive) {
-          const { info, releaseRenamed, grouped } = content.archive
-          const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
-            .filter(Boolean)
-            .join(', ')
-          this.logger.info(
-            `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
-              (grouped ? ', grouped into a release' : '') +
-              (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
-          )
-        }
-        if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
-        if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
-        if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
-        if (content?.nzb) {
-          const n = content.nzb
-          this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
-        }
+        for (const line of describeContent(file.filename, content)) this.logger.info(line)
       } catch (error: any) {
         this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
       }
@@ -66,10 +47,28 @@ export default class CollectionWorker extends BaseCommand {
       this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
     })
 
+    // Complete archive listings (D40): background work beside inspection, on the same NNTP pool.
+    const walkConcurrency = env.get('ARCHIVE_WALK_CONCURRENCY', 1)
+    const walker = walkConcurrency > 0
+      ? new Worker('archive-walk-queue', async (job) => {
+          const o = await ArchiveWalkService.walk(Number(job.data.releaseId))
+          if (o.skipped) return o
+          this.logger.info(
+            `Archive walk of release ${o.releaseId}: ${o.before} → ${o.entries} entries from ${o.segments} segments` +
+              (o.complete ? '' : ` (stopped: ${o.stopped})`) +
+              (o.renamed ? `, renamed to "${o.renamed}"` : '')
+          )
+          return o
+        }, { connection: queueConfig.connection, concurrency: walkConcurrency })
+      : null
+    walker?.on('failed', (job, err) => {
+      this.logger.error(`Archive walk job ${job?.id} failed: ${err.message}`)
+    })
+
     this.logger.info('Collection worker started and listening for jobs.')
     await Shutdown.wait()
     this.logger.info('Shutting down: finishing active jobs...')
-    await worker.close()
+    await Promise.all([worker.close(), walker?.close()])
     await NntpService.shutdown()
     await QueueService.closeAll()
   }

commands/workers/Deferred.ts

@@ -31,6 +31,7 @@ export default class DeferredWorker extends BaseCommand {
     let done = 0
     try {
       while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        try {
           const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
           if (batch.length === 0) {
             if (this.limit) break
@@ -48,6 +49,11 @@ export default class DeferredWorker extends BaseCommand {
           )
           done += batch.length
           this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+        } catch (err: any) {
+          // E.g. Postgres restarting: wait and claim again, instead of ending the loop (LOG.md 2026-10-07 (3)).
+          this.logger.error(`Round failed, retrying in 30 s: ${err.message}`)
+          await Shutdown.sleep(30_000)
+        }
       }
     } finally {
       await NntpService.shutdown()

commands/workers/Fetch.ts

@@ -12,6 +12,7 @@ import FetchRangeService from '#services/FetchRangeService'
 import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
 import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService'
 import SpotService, { isSpotGroup } from '#services/SpotService'
+import StagingService from '#services/StagingService'
 
 export default class FetchWorker extends BaseCommand {
   public static commandName = 'worker:fetch'
@@ -36,8 +37,10 @@ export default class FetchWorker extends BaseCommand {
       let failed = false
       try {
         conn = await pool.acquire()
-        await conn.group(groupName)
-        overviews = (await conn.xover(startId, endId)).overviews
+        await pool.timed(conn.group(groupName), `GROUP ${groupName}`)
+        const reply = await pool.timed<any>(conn.xover(startId, endId), `XOVER ${groupName} ${startId}-${endId}`)
+        // An empty reply parses as one line without fields ([null, {}]).
+        overviews = reply.overviews.filter(([number, header]: [number | null, Record<string, string>]) => number !== null && header?.['message-id'])
       } catch (error: any) {
         failed = true
         this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`)
@@ -56,7 +59,8 @@ export default class FetchWorker extends BaseCommand {
       }
 
       const group = await db.from('groups').where('name', groupName).first()
-      const fetchBodies = Boolean(group?.fetch_bodies)
+      // Backfill is header-only: old bodies would be far more than the link carries (CONCEPT.md D32).
+      const fetchBodies = Boolean(group?.fetch_bodies) && !job.data.backfill
 
       const segments: IngestSegment[] = []
       const obfuscated: { articleNumber: number; header: Record<string, string> }[] = []
@@ -72,7 +76,11 @@ export default class FetchWorker extends BaseCommand {
         else obfuscated.push({ articleNumber, header })
       }
 
-      const completed = await IngestService.ingest(segments)
+      // Lone segments of new files wait in staging until their file proves real (D35).
+      const { ingest, staged } = StagingService.enabled(groupName)
+        ? await StagingService.filter(segments, { counters: StagingService.countersEnabled(groupName) })
+        : { ingest: segments, staged: 0 }
+      const completed = await IngestService.ingest(ingest)
       const deferredFiles = await DeferredService.record(groupName, deferred)
 
       if (obfuscated.length > 0) {
@@ -82,7 +90,7 @@ export default class FetchWorker extends BaseCommand {
       if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
 
       this.logger.info(
-        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${segments.length} stored, ` +
+        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${ingest.length} stored${staged ? `, ${staged} staged` : ''}, ` +
           `${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` +
           `${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.`
       )

commands/workers/File.ts

@@ -1,7 +1,8 @@
 import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
-import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import BinaryFileService from '#services/BinaryFileService'
 import queueConfig from '#config/queue'
 import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
@@ -17,82 +18,28 @@ export default class FileWorker extends BaseCommand {
   public async run() {
     this.logger.info('Starting file worker...')
 
-    const collectionQueue = QueueService.collectionQueue
-
     const worker = new Worker('file-queue', async (job) => {
-      const { binaryId } = job.data
-
-      const binary = await db.from('binaries').where('id', binaryId).first()
-      if (!binary) {
-        this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`)
-        return
-      }
-
-      if (binary.conflicted_at) {
-        this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`)
-        return
-      }
-
-      const parts = await db
-        .from('parts')
-        .where('binary_id', binaryId)
-        .orderBy('number')
-        .select('number', 'message_id', 'bytes')
-      if (parts.length < binary.total_parts) {
-        throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
-      }
-
-      const messageIds = Object.fromEntries(
-        parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }])
+      // A batch of completed binaries ('process-files'), or one from before batching ('process-file').
+      const ids: number[] = (job.data.binaryIds ?? [job.data.binaryId]).map(Number)
+      const result = await BinaryFileService.store(ids)
+      if (result.saved || result.existing) {
+        this.logger.info(
+          `Files: ${result.saved} saved, ${result.existing} existed already` +
+            (result.skipped ? `, ${result.skipped} skipped` : '') +
+            (result.inspections ? `, ${result.inspections} queued for inspection` : '') +
+            ` (${ids.length} binaries).`
         )
-
-      // Already known as a file, e.g. added from a posted NZB before its own articles completed.
-      const existing = await db
-        .from('files')
-        .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? ''])
-        .first()
-      if (existing) {
-        await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
-        await dropCopiedParts(binary)
-        this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`)
-        return
       }
-
-      // file_key is unique: a second completion of the same upload inserts nothing.
-      const knex = db.connection().getWriteClient()
-      const now = new Date()
-      const [inserted] = await knex('files')
-        .insert({
-          file_key: binary.file_key,
-          filename: binary.name,
-          poster: binary.poster,
-          date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
-          parts: parts.length,
-          message_ids: JSON.stringify(messageIds),
-          groups: JSON.stringify(binary.groups),
-          release_id: binary.release_id,
-          real_name: binary.real_name,
-          size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
-          created_at: now,
-          updated_at: now,
+      // The rest is stored; the job's retry picks these up (the others are linked now and skipped).
+      if (result.incomplete.length > 0) {
+        throw new Error(`${result.incomplete.length} of ${ids.length} binaries don't have all their parts yet.`)
+      }
+    }, {
+      connection: queueConfig.connection,
+      // One job at a time could not keep up with the backfill (130k waiting); jobs are per binary and
+      // the file insert ignores duplicates, so they can run side by side.
+      concurrency: env.get('FILE_CONCURRENCY', 8),
     })
-        .onConflict('file_key')
-        .ignore()
-        .returning('id')
-
-      const fileId =
-        inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
-      await db.from('binaries').where('id', binaryId).update({ file_id: fileId })
-      await dropCopiedParts(binary)
-
-      if (!inserted) {
-        this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`)
-        return
-      }
-
-      this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
-      await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
-    }, { connection: queueConfig.connection })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`File job ${job?.id} failed: ${err.message}`)
@@ -105,13 +52,3 @@ export default class FileWorker extends BaseCommand {
     await QueueService.closeAll()
   }
 }
-
-/**
- * The file holds every message id now. Binaries identified by their subject
- * (no size: header-derived) don't need their parts any more; body-derived
- * ones keep them, so articles of an equal-sized next volume are recognised
- * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
- */
-async function dropCopiedParts(binary: { id: number; size: number | null }) {
-  if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete()
-}

commands/workers/Spots.ts

@@ -6,10 +6,17 @@ import NntpService from '#services/NntpService'
 import SpotService from '#services/SpotService'
 
 const IDLE_MS = 30_000
+/** Spots claimed at a time; most of a history backfill is listed in bulk. */
+const BATCH = 500
+/** Imported spots whose details are read per idle round. */
+const DETAILS_BATCH = 50
+/** Listed spots whose article and NZB are read per idle round (passwords, dummies). */
+const LISTINGS_BATCH = 50
 
 /**
  * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
- * downloads its compressed NZB (a few KB to a few MB) and imports it.
+ * downloads its compressed NZB (a few KB to a few MB) and imports it. When
+ * idle, reads what listed spots were listed without: details, passwords.
  */
 export default class SpotsWorker extends BaseCommand {
   public static commandName = 'worker:spots'
@@ -29,23 +36,38 @@ export default class SpotsWorker extends BaseCommand {
     let done = 0
     try {
       while (!Shutdown.requested && (!this.limit || done < this.limit)) {
-        const batch = await SpotService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        try {
+          const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
           if (batch.length === 0) {
             if (this.limit) break
-          await Shutdown.sleep(IDLE_MS)
+            // Idle: read the details of imported spots that don't have them yet (newest first), then
+            // the article and NZB of listed spots (their passwords; LOG.md 2026-10-08 (10)).
+            if ((await SpotService.fillDetails(DETAILS_BATCH)) > 0) continue
+            const listings = await SpotService.readListings(LISTINGS_BATCH, concurrency)
+            const read = Object.values(listings).reduce((sum, n) => sum + n, 0)
+            if (read === 0 || listings.retry === read) await Shutdown.sleep(IDLE_MS)
             continue
           }
+          // Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
+          const listable = batch.filter((spot) => SpotService.isListable(spot))
+          const single = batch.filter((spot) => !SpotService.isListable(spot))
+          counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
           let next = 0
           await Promise.all(
             Array.from({ length: concurrency }, async () => {
-            while (next < batch.length) {
-              const status = await SpotService.process(batch[next++])
+              while (next < single.length) {
+                const status = await SpotService.process(single[next++])
                 counts[status] = (counts[status] ?? 0) + 1
               }
             })
           )
           done += batch.length
           this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+        } catch (err: any) {
+          // E.g. Postgres restarting: wait and claim again, instead of ending the loop (LOG.md 2026-10-07 (3)).
+          this.logger.error(`Round failed, retrying in 30 s: ${err.message}`)
+          await Shutdown.sleep(30_000)
+        }
       }
     } finally {
       await NntpService.shutdown()

config/bodyparser.ts

@@ -32,6 +32,11 @@ const bodyParserConfig = defineConfig({
      */
     convertEmptyStringsToNull: true,
 
+    /**
+     * Seeding nodes post batches of a few MB (CONCEPT.md D38); the limit applies after gzip.
+     */
+    limit: '64mb',
+
     /**
      * Content types handled by the JSON parser.
      */

config/gc.ts

@@ -5,7 +5,16 @@ import env from '#start/env'
  */
 export default {
   /** How often `gc:scheduler` sweeps. */
-  intervalMs: env.get('GC_INTERVAL_MS', 10 * 60 * 1000),
+  intervalMs: env.get('GC_INTERVAL_MS', 3 * 60 * 1000),
+
+  /** How often gc:scheduler recomputes the completeness of releases that changed (LOG.md 2026-10-06 (6)). */
+  releaseCompleteIntervalMs: env.get('RELEASE_COMPLETE_INTERVAL_MS', 30 * 60 * 1000),
+
+  /** How often gc:scheduler merges uploads split per poster (D35); it reads every release. */
+  mergeIntervalMs: env.get('GC_MERGE_INTERVAL_MS', 60 * 60 * 1000),
+
+  /** Time per sweep for copying old file keys into file_keys until it's complete (LOG.md 2026-10-06 (7)). */
+  fileKeysFillMs: env.get('FILE_KEYS_FILL_SECONDS', 90) * 1000,
 
   jobs: {
     /** Completed jobs are only kept for inspection; keep them briefly. */
@@ -34,14 +43,39 @@ export default {
      * complete from headers. A real file gets its next articles in minutes.
      */
     singleArticleAfterSeconds: env.get('GC_SINGLE_ARTICLE_SECONDS', 2 * 60 * 60),
+    /**
+     * Completed header-derived binaries without a fingerprint (size, MD5 of
+     * the first 16 KiB): their file exists and no par2 set can name them
+     * later, so they go sooner than staleAfterSeconds.
+     */
+    unfingerprintedAfterSeconds: env.get('GC_UNFINGERPRINTED_SECONDS', 6 * 60 * 60),
+    /**
+     * A finished binary whose file waits for content inspection is kept this long at
+     * most (it needs the fingerprint inspection stores for par2 naming, D16).
+     */
+    awaitInspectionSeconds: env.get('GC_AWAIT_INSPECTION_SECONDS', 7 * 24 * 60 * 60),
   },
 
+  /**
+   * Staged lone segments (CONCEPT.md D35) are dropped after this long. A
+   * real file's next segment comes in the same or the next fetch range,
+   * seconds to minutes later; at backfill speed two hours of waiting
+   * segments were 12M rows, 10 GB.
+   */
+  stagingTtlSeconds: env.get('STAGING_TTL_SECONDS', 30 * 60),
+
   /** Releases without files or binaries (their binaries were collected) are deleted after this long. */
   emptyReleaseAfterSeconds: env.get('GC_EMPTY_RELEASE_SECONDS', 2 * 60 * 60),
 
-  /** Bulk deletes run in batches of this size, for at most this long per sweep. */
+  /**
+   * Bulk deletes run in batches of this size, each kind (copied parts,
+   * single-article and unfingerprinted binaries, empty releases) for at most
+   * this long per sweep, so one backlog can't starve the others.
+   */
   deleteBatch: 20_000,
-  deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 3 * 60 * 1000),
+  deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 60 * 1000),
+  /** Bloated indexes are rebuilt at most this often (IndexMaintenance). */
+  reindexIntervalMs: env.get('GC_REINDEX_INTERVAL_MS', 7 * 24 * 60 * 60 * 1000),
 
   /**
    * Deferred files (CONCEPT.md D28) are deleted this long after they were

config/indexer.ts

@@ -39,6 +39,80 @@ export default {
    * binaries. Add one with `group:add free.pt --backfill 5000`.
    */
   spotGroups: ['free.pt'],
+  /**
+   * Spots posted within this many days are imported (their NZB read and its
+   * files added). Older spots are listed: title, size and date, with the NZB
+   * read on first download.
+   */
+  spotImportDays: env.get('SPOT_IMPORT_DAYS', 30),
+
+  /**
+   * Groups whose lone header segments wait in `staged_segments` until their
+   * file shows a second one (CONCEPT.md D35). Comma-separated; `*` for all.
+   */
+  stagingGroups: (env.get('STAGING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
+
+  /**
+   * Groups whose staged segments also group by poster, file counter `[i/M]`
+   * and part count (CONCEPT.md D36), for uploads with a random subject per
+   * article. Needs staging. Comma-separated; `*` for all.
+   */
+  counterGroupingGroups: (env.get('COUNTER_GROUPING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
+
+  /**
+   * Header backfill (`group:backfill`, CONCEPT.md D32): low-priority,
+   * header-only fetch ranges walking back through a group's history.
+   */
+  seed: {
+    /** NNTP connections the seed API fetches with for relay nodes (CONCEPT.md D38); 0: relay off. */
+    nntpConnections: env.get('SEED_NNTP_CONNECTIONS', 0),
+    /**
+     * Silence that counts as a dead connection during a relay XOVER (an
+     * inactivity timeout: slow ranges keep sending). Chunks that "never
+     * answered" in 10 minutes came in 0.3 s on a fresh connection.
+     */
+    nntpTimeoutMs: env.get('SEED_NNTP_TIMEOUT_MS', 90 * 1000),
+    /**
+     * Ask the news server for gzip-compressed overviews (SEED_NNTP_COMPRESS=false: plain).
+     * On one connection plain had a shorter tail for 2026 articles, but with 20 relay
+     * connections both gave ~1.5 chunks/s, plain at many times the relay's CPU and
+     * more traffic (LOG.md 2026-10-06 (9)).
+     */
+    nntpCompress: env.get('SEED_NNTP_COMPRESS', true),
+  },
+
+  backfill: {
+    /** Backfill ranges waiting or running at a time, over all groups. */
+    maxQueued: env.get('BACKFILL_MAX_QUEUED', 40),
+    /** BullMQ priority (higher runs later; live ranges have none, so they go first). */
+    priority: 10,
+    /**
+     * No new backfill ranges while the database's NVMe part (default
+     * tablespace) or its HDD part (tablespace `hdd`, the files table) is
+     * larger than this. Checked every scheduler tick.
+     */
+    maxDefaultGb: env.get('BACKFILL_MAX_NVME_GB', 45),
+    maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
+  },
+
+  /**
+   * Dropping garbage uploads early (CONCEPT.md D42). A signature (posting tool, first
+   * group) is dropped once at least minProbes of its probed random-named releases are
+   * known and at most maxNamedRate of them got a real name or password; its
+   * random-named files are then not stored, except explorePercent of its releases.
+   */
+  garbage: {
+    /** Off: nothing is filtered or deleted (the learning still runs). */
+    enabled: env.get('GARBAGE_FILTER', true),
+    explorePercent: env.get('GARBAGE_EXPLORE_PERCENT', 2),
+    minProbes: 100,
+    maxNamedRate: 0.005,
+    /** Probed releases learned from: 1 in this many (by release id), for the last windowDays. */
+    probeSample: 4,
+    windowDays: 30,
+    /** How often gc:scheduler learns the signatures again. */
+    learnIntervalMs: env.get('GARBAGE_LEARN_INTERVAL_MS', 3 * 60 * 60 * 1000),
+  },
 
   /** worker:deferred leaves a file alone until it has had no new articles for this long. */
   deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),

config/nntp.ts

@@ -8,4 +8,8 @@ export default {
   secure: env.get('NNTP_SECURE', false),
   /** Connections per process. Mind the provider's limit across all processes. */
   poolSize: env.get('NNTP_POOL_SIZE', 10),
+  /** A command without an answer this long counts as a dead connection (XOVER of 20k headers takes ~0.2 s). */
+  commandTimeoutMs: env.get('NNTP_COMMAND_TIMEOUT_MS', 120_000),
+  /** TLS certificate hostname when NNTP_HOST is a tunnel (seeding nodes; lib/nntp/xover_client). */
+  tlsServername: env.get('NNTP_TLS_SERVERNAME'),
 }

database/migrations/1768620765010_spot_listings.ts

@@ -22,11 +22,16 @@ export default class extends BaseSchema {
 
   async up() {
     await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS verified boolean`)
+    // worker:spots claims recent spots first, then the history in posting order.
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
     await this.swap(VISIBLE)
   }
 
   async down() {
     await this.swap('file_count > 0')
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_article_idx`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_posted_idx`)
     await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS verified`)
   }
 

database/migrations/1768620765020_spot_details.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Spot details (CONCEPT.md D29): the description and image from the spot's
+ * full header. Stored whenever the header is read (recent spots, moderation,
+ * downloads); for other old spots on first view. `details_at` marks the read.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('spots', (table) => {
+      table.text('description')
+      table.jsonb('image')
+      table.timestamp('details_at', { useTz: true })
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('spots', (table) => {
+      table.dropColumn('description')
+      table.dropColumn('image')
+      table.dropColumn('details_at')
+    })
+  }
+}

database/migrations/1768620765030_release_password_source.ts

@@ -0,0 +1,16 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Where a release's archive password came from: `nzb` (the NZB's
+ * `<meta type="password">`) or `spot` (the spot's title or description).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('releases', (table) => table.string('password_source', 8))
+    this.schema.raw(`UPDATE releases SET password_source = 'nzb' WHERE password IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => table.dropColumn('password_source'))
+  }
+}

database/migrations/1768620765040_pack_file_segments.ts

@@ -0,0 +1,63 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import { packSegments, unpackSegments } from '#lib/nzb/segments'
+
+const BATCH = 5000
+
+/**
+ * Segment lists compressed (CONCEPT.md D31): `files.message_ids` (jsonb,
+ * ~38 bytes per segment after pglz) becomes `files.segments` (zstd, ~20),
+ * plus `first_message_id` for the lookups the old expression index served.
+ * Run with the workers stopped; reclaim the space with VACUUM FULL files.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS segments bytea, ADD COLUMN IF NOT EXISTS first_message_id text`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await this.db.rawQuery(
+        `SELECT id, message_ids FROM files WHERE id > ? AND segments IS NULL ORDER BY id LIMIT ?`,
+        [last, BATCH]
+      )
+      if (rows.length === 0) break
+      const ids: number[] = []
+      const packed: Buffer[] = []
+      const firsts: (string | null)[] = []
+      for (const row of rows) {
+        const segments = Object.entries((row.message_ids ?? {}) as Record<string, { id: string; size?: number }>)
+          .map(([number, s]) => ({ number: Number(number), id: s.id, size: Number(s.size ?? 0) }))
+          .sort((a, b) => a.number - b.number)
+        ids.push(Number(row.id))
+        packed.push(packSegments(segments))
+        firsts.push(segments[0]?.id ?? null)
+      }
+      await this.db.rawQuery(
+        `UPDATE files f SET segments = v.segments, first_message_id = v.first
+         FROM unnest(?::bigint[], ?::bytea[], ?::text[]) AS v(id, segments, first) WHERE f.id = v.id`,
+        [ids, packed, firsts] as any
+      )
+      last = ids[ids.length - 1]
+      done += rows.length
+      if (done % 100_000 < BATCH) console.log(`Packed the segments of ${done} files.`)
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN segments SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_first_message_id_idx ON files (first_message_id)`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_first_segment_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN message_ids`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS message_ids jsonb`)
+    const { rows } = await this.db.rawQuery(`SELECT id, segments FROM files`)
+    for (const row of rows) {
+      const json = Object.fromEntries(unpackSegments(row.segments).map((s) => [s.number, { id: s.id, size: s.size }]))
+      await this.db.rawQuery(`UPDATE files SET message_ids = ? WHERE id = ?`, [JSON.stringify(json), row.id])
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN message_ids SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
+    await this.db.rawQuery(`DROP INDEX IF EXISTS files_first_message_id_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN segments, DROP COLUMN first_message_id`)
+  }
+}

database/migrations/1768620765050_group_backfill.ts

@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Header backfill (CONCEPT.md D32): a group walks backwards from
+ * `backfill_next_id` down to `backfill_until_id` in low-priority,
+ * header-only fetch ranges (`fetch_ranges.backfill`).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('groups', (table) => {
+      table.bigInteger('backfill_until_id')
+      table.bigInteger('backfill_next_id')
+    })
+    this.schema.alterTable('fetch_ranges', (table) => {
+      table.boolean('backfill').notNullable().defaultTo(false)
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('groups', (table) => {
+      table.dropColumn('backfill_until_id')
+      table.dropColumn('backfill_next_id')
+    })
+    this.schema.alterTable('fetch_ranges', (table) => table.dropColumn('backfill'))
+  }
+}

database/migrations/1768620765060_search_text_and_index_cleanup.ts

@@ -0,0 +1,144 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+const VISIBLE = `(file_count > 0 OR source = 'spot')`
+const BATCH = 20_000
+
+/**
+ * Search and index cleanup (CONCEPT.md D33).
+ * - `releases.search_text`: name, archive listing and readable file-name
+ *   stems, normalised (lowercase; dots, dashes, underscores and spaces as
+ *   one space), kept by triggers; one trigram index replaces the four the
+ *   old per-table OR search could hardly use.
+ * - Spot status indexes become partial indexes over the few rows the worker
+ *   and search look for.
+ * - Unused release indexes go (size: never scanned; complete: a boolean).
+ * Runs online: indexes are built and dropped concurrently.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
+    // For IndexMaintenance; needs a superuser (it is one in the compose setup).
+    await q(`CREATE EXTENSION IF NOT EXISTS pgstattuple`).catch((error) => console.warn(`pgstattuple not installed: ${error.message}`))
+
+    // Spots: the worker only looks for pending/processing and un-detailed imported spots, search for hidden ones.
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_todo_idx ON spots (article_number) WHERE status IN ('pending', 'processing')`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_hidden_release_idx ON spots (release_id) WHERE status IN ('removed', 'dummy')`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_details_todo_idx ON spots (posted_at DESC) WHERE status = 'imported' AND details_at IS NULL`)
+    for (const index of ['spots_status_posted_idx', 'spots_status_article_idx', 'spots_status_created_at_index', 'spots_status_release_id_index']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+    for (const index of ['releases_size_index', 'releases_complete_idx']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+
+    // Search text.
+    await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS search_files text, ADD COLUMN IF NOT EXISTS search_text text`)
+    await q(`
+      CREATE OR REPLACE FUNCTION search_normalize(t text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT btrim(lower(regexp_replace(coalesce(t, ''), '[._[:space:]-]+', ' ', 'g')))
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION release_search_text(name text, contents jsonb, files text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT left(concat_ws(' ',
+          search_normalize(name),
+          nullif(search_normalize((SELECT string_agg(e->>'name', ' ') FROM (
+            SELECT e FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) e LIMIT 200) x)), ''),
+          nullif(files, '')), 8000)
+      $$`)
+    // Visible releases only; computed when one becomes visible or its name, listing or file names change.
+    await q(`
+      CREATE OR REPLACE FUNCTION releases_search_text() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        IF NOT (NEW.file_count > 0 OR NEW.source = 'spot') THEN
+          NEW.search_text := NULL;
+        ELSIF TG_OP = 'INSERT' OR NEW.search_text IS NULL
+          OR NEW.name IS DISTINCT FROM OLD.name OR NEW.contents IS DISTINCT FROM OLD.contents
+          OR NEW.search_files IS DISTINCT FROM OLD.search_files THEN
+          NEW.search_text := release_search_text(NEW.name, NEW.contents, NEW.search_files);
+        END IF;
+        RETURN NEW;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
+    await q(`CREATE TRIGGER releases_search_text BEFORE INSERT OR UPDATE ON releases FOR EACH ROW EXECUTE FUNCTION releases_search_text()`)
+    // A file's readable stem joins its release's search text when it adds something.
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_search() RETURNS trigger LANGUAGE plpgsql AS $$
+      DECLARE
+        n text := coalesce(NEW.real_name, NEW.filename);
+        s text;
+      BEGIN
+        IF NEW.release_id IS NULL OR NOT is_readable_name(n) THEN RETURN NULL; END IF;
+        s := search_normalize(name_stem(n));
+        IF s = '' THEN RETURN NULL; END IF;
+        UPDATE releases SET search_files = left(concat_ws(' ', search_files, s), 4000)
+        WHERE id = NEW.release_id AND position(s IN coalesce(search_text, search_normalize(name))) = 0
+          AND length(coalesce(search_files, '')) < 4000;
+        RETURN NULL;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
+    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+
+    // Fill: readable file stems per release (one pass over files), then every visible release in batches.
+    await q(`DROP TABLE IF EXISTS search_stems_tmp`)
+    await q(`
+      CREATE UNLOGGED TABLE search_stems_tmp AS
+      SELECT release_id, left(string_agg(DISTINCT s, ' '), 4000) AS stems FROM (
+        SELECT f.release_id, search_normalize(name_stem(coalesce(f.real_name, f.filename))) AS s
+        FROM files f WHERE f.release_id IS NOT NULL AND is_readable_name(coalesce(f.real_name, f.filename))) x
+      WHERE s <> '' GROUP BY release_id`)
+    await q(`CREATE UNIQUE INDEX ON search_stems_tmp (release_id)`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
+      if (rows.length === 0) break
+      const to = Number(rows[rows.length - 1].id)
+      await q(
+        `UPDATE releases r SET search_files = (
+           SELECT nullif(array_to_string(array(
+             SELECT w FROM unnest(string_to_array(t.stems, ' ')) w
+             WHERE position(w IN search_normalize(r.name)) = 0), ' '), '')
+           FROM search_stems_tmp t WHERE t.release_id = r.id),
+           search_text = NULL
+         WHERE r.id > ? AND r.id <= ? AND ${VISIBLE.replaceAll('file_count', 'r.file_count').replace('source', 'r.source')}`,
+        [last, to]
+      )
+      last = to
+      done += rows.length
+      if (done % 200_000 < BATCH) console.log(`Search text for ${done} releases.`)
+    }
+    await q(`DROP TABLE search_stems_tmp`)
+
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_search_trgm ON releases USING gin (search_text gin_trgm_ops) WHERE ${VISIBLE}`)
+    for (const index of ['releases_name_trgm', 'releases_contents_trgm', 'files_filename_trgm', 'files_real_name_trgm']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+  }
+
+  async down() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_search_trgm`)
+    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
+    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
+    await q(`DROP FUNCTION IF EXISTS files_release_search()`)
+    await q(`DROP FUNCTION IF EXISTS releases_search_text()`)
+    await q(`DROP FUNCTION IF EXISTS release_search_text(text, jsonb, text)`)
+    await q(`DROP FUNCTION IF EXISTS search_normalize(text)`)
+    await q(`ALTER TABLE releases DROP COLUMN IF EXISTS search_files, DROP COLUMN IF EXISTS search_text`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_size_index ON releases (size) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_complete_idx ON releases (complete) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_created_at_index ON spots (status, created_at)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_release_id_index ON spots (status, release_id)`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_todo_idx`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_hidden_release_idx`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_details_todo_idx`)
+  }
+}

database/migrations/1768620765070_readable_name_more_shapes.ts

@@ -0,0 +1,43 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * is_readable_name(): also hex with a Unix time, UUIDs, digit-only names
+ * (and rclone's "83111763 n"), lowercase base32/36 tokens and short tokens
+ * with capitals all over. The same rules as looksRandom() in ArchiveService.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      await db.rawQuery(`
+        CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+          WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+          SELECT name IS NOT NULL AND NOT (
+            (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+              AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+            OR stem ~* '^[0-9a-f]{16,}$'
+            OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
+            OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+            OR stem ~ '^\\d{6,}( n){0,1}$'
+            OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
+                AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
+            OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
+                AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3))
+          FROM s
+        $$`)
+    })
+  }
+
+  async down() {
+    this.defer(async (db) => {
+      await db.rawQuery(`
+        CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+          WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+          SELECT name IS NOT NULL AND NOT (
+            (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+              AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+            OR stem ~* '^[0-9a-f]{16,}$')
+          FROM s
+        $$`)
+    })
+  }
+}

database/migrations/1768620765080_create_sfv_entries.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * SFV listings (CONCEPT.md D34): the files an SFV names belong to its
+ * release; files arriving later are matched by name.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.createTable('sfv_entries', (table) => {
+      table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE')
+      table.text('name').notNullable()
+      table.string('crc', 8).notNullable()
+      table.text('poster')
+      table.timestamp('posted_at', { useTz: true })
+      table.primary(['release_id', 'name'])
+      table.index(['name'])
+    })
+  }
+
+  async down() {
+    this.schema.dropTable('sfv_entries')
+  }
+}

database/migrations/1768620765090_files_name_index.ts

@@ -0,0 +1,15 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** SFV grouping (CONCEPT.md D34) looks files up by their name. */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    // On the NVMe (default tablespace), like the files table's other indexes.
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_name_idx ON files ((coalesce(real_name, filename)))`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_name_idx`)
+  }
+}

database/migrations/1768620765100_readable_name_generic.ts

@@ -0,0 +1,37 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * is_readable_name(): generic folder/file names (VIDEO_TS, Sample, CD1) are
+ * no name either. Same list as GENERIC_NAMES in lib/release/release_key.ts.
+ */
+const FUNCTION = (generic: boolean) => `
+  CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+    WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+    SELECT name IS NOT NULL AND NOT (
+      (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+        AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+      OR stem ~* '^[0-9a-f]{16,}$'
+      OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
+      OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+      OR stem ~ '^\\d{6,}( n){0,1}$'
+      OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
+          AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
+      OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
+          AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3)
+      ${generic ? `OR btrim(stem) ~* '^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples{0,1}|subs{0,1}|subtitles{0,1}|proof|cover|covers|extras{0,1}|featurettes{0,1}|(cd|dvd|disc|disk)[ ._-]{0,1}\\d{1,2})$'` : ''})
+    FROM s
+  $$`
+
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      await db.rawQuery(FUNCTION(true))
+    })
+  }
+
+  async down() {
+    this.defer(async (db) => {
+      await db.rawQuery(FUNCTION(false))
+    })
+  }
+}

database/migrations/1768620765110_create_staged_segments.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Staged segments (CONCEPT.md D35): a header segment whose file is new and
+ * alone in its batch waits here until a second segment of the file shows
+ * up; ~97% never do (uploads with a random subject per article). Unlogged:
+ * no WAL, emptied by a crash, which only costs those waiting segments.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE UNLOGGED TABLE staged_segments (
+        file_key text PRIMARY KEY,
+        segments jsonb NOT NULL,
+        staged_at timestamptz NOT NULL
+      )`)
+    // Rows arrive in time order, so a BRIN index serves the expiry cheaply.
+    await this.db.rawQuery(`CREATE INDEX staged_segments_staged_at_brin ON staged_segments USING brin (staged_at)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE staged_segments`)
+  }
+}

database/migrations/1768620765120_staged_segments_counter_key.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Counter grouping (CONCEPT.md D36): staged segments also carry the key of
+ * the file they'd belong to by poster, file counter `[i/M]`, part count and
+ * posting day, so uploads with a random subject per article become files.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE staged_segments ADD COLUMN IF NOT EXISTS counter_key text`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS staged_segments_counter_key_idx ON staged_segments (counter_key) WHERE counter_key IS NOT NULL`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS staged_segments_counter_key_idx`)
+    await this.db.rawQuery(`ALTER TABLE staged_segments DROP COLUMN IF EXISTS counter_key`)
+  }
+}

database/migrations/1768620765130_release_extensions.ts

@@ -0,0 +1,109 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+const VISIBLE = `(file_count > 0 OR source = 'spot')`
+const BATCH = 20_000
+
+/**
+ * `releases.extensions`: the file types in a release (`{mkv,nfo,par2}`), from
+ * its files' names and its archive listing, for the search's extension
+ * filter (LOG.md 2026-10-04 (3)). search_text can't serve it: file names
+ * enter it as stems without extension, and matching the long, TOASTed text
+ * took seconds per search.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
+
+    await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS extensions text[]`)
+    // Two to four characters with a letter (mkv, flac, 7z, par2); split
+    // archive volumes count as their archive type (.7z.001, .r00).
+    await q(`
+      CREATE OR REPLACE FUNCTION name_extension(n text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT CASE
+          WHEN lower(n) ~ '\\.(7z|zip)\\.[0-9]{3}$' THEN substring(lower(n) from '\\.(7z|zip)\\.[0-9]{3}$')
+          WHEN lower(n) ~ '\\.r[0-9]{2,3}$' THEN 'rar'
+          ELSE substring(lower(n) from '\\.([a-z0-9]*[a-z][a-z0-9]*)$')
+        END
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION listing_extensions(contents jsonb) RETURNS text[] IMMUTABLE LANGUAGE sql AS $$
+        SELECT array(SELECT DISTINCT e FROM (
+          SELECT name_extension(c->>'name') AS e
+          FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) c) x
+        WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 ORDER BY e)
+      $$`)
+
+    // A new archive listing adds its entries' types.
+    await q(`
+      CREATE OR REPLACE FUNCTION releases_extensions() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        IF TG_OP = 'INSERT' OR NEW.contents IS DISTINCT FROM OLD.contents THEN
+          NEW.extensions := nullif(array(
+            SELECT DISTINCT e FROM unnest(coalesce(NEW.extensions, '{}') || listing_extensions(NEW.contents)) e ORDER BY e), '{}');
+        END IF;
+        RETURN NEW;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
+    await q(`CREATE TRIGGER releases_extensions BEFORE INSERT OR UPDATE OF contents ON releases FOR EACH ROW EXECUTE FUNCTION releases_extensions()`)
+
+    // A file adds its type to its release (once: most files of a release share it).
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_extension() RETURNS trigger LANGUAGE plpgsql AS $$
+      DECLARE
+        e text := name_extension(coalesce(NEW.real_name, NEW.filename));
+      BEGIN
+        IF NEW.release_id IS NULL OR e IS NULL OR length(e) NOT BETWEEN 2 AND 4 THEN RETURN NULL; END IF;
+        UPDATE releases SET extensions = array(SELECT DISTINCT x FROM unnest(coalesce(extensions, '{}') || e) x ORDER BY x)
+        WHERE id = NEW.release_id AND NOT (coalesce(extensions, '{}') @> ARRAY[e]);
+        RETURN NULL;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
+    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+
+    // Fill: file types per release in one sequential pass over files (random
+    // reads per release would take over an hour on the HDD), then the
+    // visible releases in batches by id, adding their listing's types.
+    await q(`DROP TABLE IF EXISTS file_extensions_tmp`)
+    await q(`
+      CREATE UNLOGGED TABLE file_extensions_tmp AS
+      SELECT release_id, array_agg(DISTINCT e) AS exts FROM (
+        SELECT release_id, name_extension(coalesce(real_name, filename)) AS e FROM files WHERE release_id IS NOT NULL) x
+      WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 GROUP BY release_id`)
+    await q(`CREATE UNIQUE INDEX ON file_extensions_tmp (release_id)`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
+      if (rows.length === 0) break
+      const to = Number(rows[rows.length - 1].id)
+      await q(
+        `UPDATE releases r SET extensions = x.exts FROM (
+           SELECT r2.id, array(SELECT DISTINCT e FROM unnest(coalesce(t.exts, '{}') || listing_extensions(r2.contents)) e ORDER BY e) AS exts
+           FROM releases r2 LEFT JOIN file_extensions_tmp t ON t.release_id = r2.id
+           WHERE r2.id > ? AND r2.id <= ? AND (r2.file_count > 0 OR r2.source = 'spot')) x
+         WHERE r.id = x.id AND cardinality(x.exts) > 0 AND r.extensions IS DISTINCT FROM x.exts`,
+        [last, to]
+      )
+      last = to
+      done += rows.length
+      if (done % 500_000 < BATCH) console.log(`Extensions for ${done} releases.`)
+    }
+    await q(`DROP TABLE file_extensions_tmp`)
+
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_extensions_gin ON releases USING gin (extensions) WHERE ${VISIBLE}`)
+  }
+
+  async down() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_extensions_gin`)
+    await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
+    await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
+    await q(`DROP FUNCTION IF EXISTS files_release_extension()`)
+    await q(`DROP FUNCTION IF EXISTS releases_extensions()`)
+    await q(`DROP FUNCTION IF EXISTS listing_extensions(jsonb)`)
+    await q(`DROP FUNCTION IF EXISTS name_extension(text)`)
+    await q(`ALTER TABLE releases DROP COLUMN IF EXISTS extensions`)
+  }
+}

database/migrations/1768620765140_seed_orchestrator.ts

@@ -0,0 +1,47 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The seed orchestrator (CONCEPT.md D38): nodes that lease article ranges
+ * over HTTPS and submit what they aggregated (D37), and the ranges planned
+ * for them. A range's id is its priority: plans insert newest first.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.createTable('seed_nodes', (table) => {
+      table.increments('id')
+      table.text('name').notNullable().unique()
+      // sha256 of the bearer token; the token itself is shown once, when it's made.
+      table.string('token_hash', 64).notNullable().unique()
+      table.boolean('disabled').notNullable().defaultTo(false)
+      table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
+      table.timestamp('last_seen_at', { useTz: true }).nullable()
+    })
+
+    this.schema.createTable('seed_ranges', (table) => {
+      table.bigIncrements('id')
+      table.text('group_name').notNullable()
+      table.bigInteger('start_id').notNullable()
+      table.bigInteger('end_id').notNullable()
+      // First article not reported yet; a re-leased range resumes here.
+      table.bigInteger('next_id').notNullable()
+      table.string('status', 8).notNullable().defaultTo('queued')
+      table.integer('node_id').nullable().references('id').inTable('seed_nodes').onDelete('SET NULL')
+      table.integer('attempts').notNullable().defaultTo(0)
+      table.timestamp('leased_at', { useTz: true }).nullable()
+      table.timestamp('heartbeat_at', { useTz: true }).nullable()
+      table.timestamp('done_at', { useTz: true }).nullable()
+      table.bigInteger('headers').notNullable().defaultTo(0)
+      table.bigInteger('files').notNullable().defaultTo(0)
+      table.bigInteger('fragments').notNullable().defaultTo(0)
+      table.text('error').nullable()
+      table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
+      table.index(['group_name', 'start_id'])
+    })
+    this.schema.raw(`CREATE INDEX seed_ranges_open_idx ON seed_ranges (id) WHERE status IN ('queued', 'leased')`)
+  }
+
+  async down() {
+    this.schema.dropTable('seed_ranges')
+    this.schema.dropTable('seed_nodes')
+  }
+}

database/migrations/1768620765150_partition_files.ts

@@ -0,0 +1,77 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** Ids per partition: 4M files is a few days of indexing, ~2 GB of heap (FilePartitions uses the same). */
+const FILE_PARTITION_SIZE = 4_000_000
+
+/**
+ * `files` partitioned by id, i.e. by age (CONCEPT.md D39). New rows went into
+ * free space scattered through the 9.6 GB heap on the HDD (left by
+ * inspection's updates), one random read per insert; now they go into a
+ * small recent partition that stays in memory. The existing table becomes
+ * the first partition as it is: only a CHECK scan, no rewrite.
+ *
+ * Postgres can't enforce a unique file_key across partitions: every
+ * partition has its own unique index, and FileStore.insert looks keys up in
+ * the older ones. Run with the writers stopped (a few minutes).
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    const { rows } = await q(`SELECT coalesce(max(id), 0) AS max FROM files`)
+    // The first new partition starts at the next million.
+    const boundary = (Math.floor(Number(rows[0].max) / 1_000_000) + 1) * 1_000_000
+    // The HDD tablespace on homelab (D32), the default elsewhere; new partitions inherit the parent's.
+    const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`)
+    const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : ''
+
+    // The old table, renamed; its constraint and index names get a files_p0 prefix.
+    await q(`ALTER TABLE files ADD CONSTRAINT files_p0_bound CHECK (id < ${boundary}) NOT VALID`)
+    await q(`ALTER TABLE files VALIDATE CONSTRAINT files_p0_bound`)
+    await q(`ALTER TABLE files RENAME TO files_p0`)
+    await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_pkey TO files_p0_pkey`)
+    await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_file_key_unique TO files_p0_file_key_unique`)
+    await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_release_id_foreign TO files_p0_release_id_foreign`)
+    for (const index of ['files_content_type_index', 'files_first_message_id_idx', 'files_name_idx', 'files_release_id_index']) {
+      await q(`ALTER INDEX ${index} RENAME TO ${index.replace(/^files_/, 'files_p0_')}`)
+    }
+    // Recreated on the parent, which clones them to every partition.
+    for (const trigger of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
+      await q(`DROP TRIGGER ${trigger} ON files_p0`)
+    }
+
+    // The parent: same columns and defaults (the id sequence), partitioned by id, in the old table's tablespace.
+    await q(`CREATE TABLE files (LIKE files_p0 INCLUDING DEFAULTS) PARTITION BY RANGE (id)${tablespace}`)
+    await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`)
+    await q(`CREATE INDEX files_content_type_index ON files (content_type)`)
+    await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`)
+    await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`)
+    await q(`CREATE INDEX files_release_id_index ON files (release_id)`)
+    // Matching indexes and the foreign key of files_p0 are taken over, not rebuilt; the CHECK spares the scan.
+    await q(`ALTER TABLE files ATTACH PARTITION files_p0 FOR VALUES FROM (MINVALUE) TO (${boundary})`)
+    await q(`ALTER TABLE files_p0 DROP CONSTRAINT files_p0_bound`)
+
+    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+    await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
+    await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
+
+    // binaries.file_id followed the rename to files_p0; point it at the parent (no scan needed).
+    await q(`ALTER TABLE binaries DROP CONSTRAINT binaries_file_id_foreign`)
+    await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`)
+
+    // The next two partitions; FilePartitions.ensure() (gc:scheduler) keeps two ahead from now on.
+    for (const start of [boundary, boundary + FILE_PARTITION_SIZE]) {
+      const name = `files_p${start}`
+      await q(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${start}) TO (${start + FILE_PARTITION_SIZE})`)
+      await q(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
+    }
+  }
+
+  async down() {
+    throw new Error('Not reversible: merging partitions back means rewriting the files table.')
+  }
+}

database/migrations/1768620765160_files_bigint.ts

@@ -0,0 +1,89 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+const TRIGGERS = [
+  `files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`,
+  `files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`,
+  `files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`,
+  `files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`,
+]
+
+/**
+ * files.id (and binaries.file_id, the id sequence) to 64 bits (CONCEPT.md
+ * D39): seeding adds ~14M files a day, and 32 bits run out at 2.1 billion.
+ * Postgres can't change the type of a partition key, so the partitions are
+ * detached, rewritten one by one as plain tables (files_p0: ~100 GB with its
+ * segment lists, 1–2 hours on homelab's HDD) and attached to a new 64-bit
+ * parent. A CHECK per partition, added before its rewrite, spares the attach
+ * scan. Run with the writers stopped and Postgres on wal_level=minimal, or the
+ * rewrite also goes through the WAL (LOG.md 2026-10-05 (8)).
+ */
+export default class extends BaseSchema {
+  // One transaction: a failure anywhere leaves files as it was (it drops and recreates the parent).
+
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    const { rows: partitions } = await q(
+      `SELECT c.relname AS name, pg_get_expr(c.relpartbound, c.oid) AS bound
+       FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname`
+    )
+    const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`)
+    const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : ''
+
+    // The sequence belongs to the old parent: keep it when that goes.
+    await q(`ALTER SEQUENCE files_id_seq OWNED BY NONE`)
+    await q(`ALTER SEQUENCE files_id_seq AS bigint`)
+    await q(`ALTER TABLE binaries DROP CONSTRAINT IF EXISTS binaries_file_id_foreign`)
+    await q(`ALTER TABLE binaries ALTER COLUMN file_id TYPE bigint`)
+
+    for (const p of partitions) {
+      await q(`ALTER TABLE files DETACH PARTITION ${p.name}`)
+      // Triggers cloned from the parent may stay behind on a detached partition; the new parent clones them again.
+      for (const t of TRIGGERS) await q(`DROP TRIGGER IF EXISTS ${t.split(' ')[0]} ON ${p.name}`)
+      const [, from, to] = /FROM \((.+)\) TO \((.+)\)/.exec(p.bound.replaceAll("'", ''))!
+      const conditions = [from === 'MINVALUE' ? null : `id >= ${from}`, to === 'MAXVALUE' ? null : `id < ${to}`].filter(Boolean)
+      if (conditions.length > 0) await q(`ALTER TABLE ${p.name} ADD CONSTRAINT ${p.name}_bound CHECK (${conditions.join(' AND ')})`)
+    }
+    await q(`DROP TABLE files`)
+
+    for (const p of partitions) {
+      await q(`ALTER TABLE ${p.name} ALTER COLUMN id TYPE bigint`)
+    }
+
+    await q(`
+      CREATE TABLE files (
+        id bigint NOT NULL DEFAULT nextval('files_id_seq'),
+        filename varchar(255) NOT NULL,
+        poster varchar(255) NOT NULL,
+        date bigint NOT NULL,
+        parts integer NOT NULL,
+        groups jsonb,
+        created_at timestamptz,
+        updated_at timestamptz,
+        file_key varchar(40),
+        release_id bigint,
+        archive jsonb,
+        real_name text,
+        size bigint,
+        content_type varchar(16),
+        segments bytea NOT NULL,
+        first_message_id text
+      ) PARTITION BY RANGE (id)${tablespace}`)
+    await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`)
+    await q(`CREATE INDEX files_content_type_index ON files (content_type)`)
+    await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`)
+    await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`)
+    await q(`CREATE INDEX files_release_id_index ON files (release_id)`)
+    for (const p of partitions) {
+      await q(`ALTER TABLE files ATTACH PARTITION ${p.name} ${p.bound}`)
+      await q(`ALTER TABLE ${p.name} DROP CONSTRAINT IF EXISTS ${p.name}_bound`)
+    }
+    for (const t of TRIGGERS) await q(`CREATE TRIGGER ${t}`)
+    await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`)
+  }
+
+  async down() {
+    throw new Error('Not reversible: ids above 2^31 may exist.')
+  }
+}

database/migrations/1768620765170_release_contents_walked.ts

@@ -0,0 +1,14 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** When an archive walk (CONCEPT.md D40) looked at a release, so it isn't walked again. */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('releases', (table) => {
+      table.timestamp('contents_walked_at', { useTz: true })
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => table.dropColumn('contents_walked_at'))
+  }
+}

database/migrations/1768620765180_release_complete_faster.ts

@@ -0,0 +1,69 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Release completeness (D24) without cubic cost (LOG.md 2026-10-06 (4)). A
+ * par2 set listing n files was inserted into release_files row by row, each
+ * row's trigger ran release_complete() twice, and that matched every listed
+ * file against all binaries in the database: n × 2 × n × (all binaries). One
+ * insert of a few thousand rows ran for over 20 minutes at full CPU.
+ *
+ * - release_complete(): the release's files are read once, by release id,
+ *   and each listed file is matched against those by size or by name. The
+ *   correlated subquery read every binary in the database per listed file
+ *   (binaries_size_index), filtering by release only afterwards.
+ * - refresh_release_complete(): computes it once.
+ * - release_files: one refresh per release per INSERT statement.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
+        WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
+        -- The release's files, once, by index (read only when there's a par2 listing to check).
+        have AS MATERIALIZED (
+          SELECT b.size, f.real_name FROM files f JOIN binaries b ON b.file_id = f.id WHERE f.release_id = rid)
+        SELECT coalesce((
+          SELECT CASE
+            WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
+            WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
+              SELECT 1 FROM release_files rf
+              WHERE rf.release_id = rid AND rf.size >= 65536
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.real_name = rf.name))
+            ELSE false END
+          FROM r), false)
+      $$`)
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        UPDATE releases r SET complete = c.complete
+        FROM (SELECT release_complete(rid) AS complete) c
+        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
+      $$`)
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION release_files_release_complete_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          PERFORM refresh_release_complete(n.release_id) FROM (SELECT DISTINCT release_id FROM new_rows) n;
+          RETURN NULL;
+        END $$`)
+    // Swapping the trigger locks release_files: inserts still running the old cubic check
+    // (minutes) are cancelled rather than waited for; their jobs are retried, and run fast.
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
+    await this.db.rawQuery(`
+      SELECT pg_cancel_backend(pid) FROM pg_stat_activity
+      WHERE pid <> pg_backend_pid() AND state = 'active' AND query ILIKE 'insert into "release_files"%'
+        AND now() - query_start > interval '30 seconds'`)
+    await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
+    await this.db.rawQuery(`
+      CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
+      REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT
+      EXECUTE FUNCTION release_files_release_complete_statement()`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
+    await this.db.rawQuery(`
+      CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
+      FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
+    await this.db.rawQuery(`DROP FUNCTION IF EXISTS release_files_release_complete_statement()`)
+  }
+}

database/migrations/1768620765190_release_totals_per_statement.ts

@@ -0,0 +1,110 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Cheaper release bookkeeping for file inserts (LOG.md 2026-10-06 (6)).
+ *
+ * Every inserted file fired four row triggers, each updating its release:
+ * file_count/size, search_files, extensions and completeness, so a seed
+ * batch of thousands of files rewrote its releases thousands of times (a
+ * quarter of Postgres' active time on homelab). Now one statement-level
+ * trigger updates each release once per INSERT, with all four.
+ *
+ * Completeness (D24), the expensive part, is no longer computed in triggers:
+ * refresh_release_complete() only notes the release in release_dirty, and
+ * gc:scheduler recomputes the noted releases every 30 minutes
+ * (recompute_release_complete). Counts, sizes, search text and extensions
+ * stay current: code decides on them right away (weak releases, deleting
+ * empty releases).
+ *
+ * Also an index on release_files (release_id, size) for the par2 size
+ * match's "listed once in its set" check (IngestService.matchPar2Releases).
+ */
+export default class extends BaseSchema {
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    // Swapping triggers locks files and its partitions. Taken first and in one statement: a
+    // DROP TRIGGER later in the transaction deadlocked with ingest (2026-10-06). Give up rather
+    // than queue every writer behind it for long.
+    await q(`SET LOCAL lock_timeout = '2min'`)
+    await q(`LOCK TABLE files, release_files IN ACCESS EXCLUSIVE MODE`)
+    // Built CONCURRENTLY by hand on homelab (2026-10-06); IF NOT EXISTS keeps that one.
+    await q(`CREATE INDEX IF NOT EXISTS release_files_release_size_idx ON release_files (release_id, size)`)
+
+    // Append-only (no key): writers never wait on each other or on the sweep.
+    await q(`CREATE TABLE IF NOT EXISTS release_dirty (release_id bigint NOT NULL)`)
+    await q(`
+      CREATE OR REPLACE FUNCTION recompute_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        UPDATE releases r SET complete = c.complete
+        FROM (SELECT release_complete(rid) AS complete) c
+        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
+      $$`)
+    // Every trigger that called this (files, release_files, binaries.size, releases.file_total) now just notes the release.
+    await q(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL
+      $$`)
+
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          WITH added AS (
+            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
+                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
+                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
+            FROM (
+              SELECT release_id, size, name_extension(nm) AS e,
+                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
+              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
+            ) x
+            GROUP BY release_id
+          )
+          UPDATE releases r SET
+            file_count = r.file_count + a.n,
+            size = r.size + a.bytes,
+            extensions = CASE
+              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
+              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
+            search_files = CASE
+              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
+              ELSE nullif(left(concat_ws(' ', r.search_files, (
+                SELECT string_agg(w, ' ') FROM unnest(a.words) w
+                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
+          FROM added a
+          WHERE r.id = a.release_id;
+
+          INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
+          RETURN NULL;
+        END $$`)
+
+    // The row triggers keep UPDATE and DELETE (renames, moves between releases); INSERT is the statement trigger.
+    for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
+      await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
+    }
+    await q(`CREATE TRIGGER files_release_extension AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+    await q(`CREATE TRIGGER files_release_search AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+    await q(`CREATE TRIGGER files_release_totals AFTER DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
+    await q(`CREATE TRIGGER files_zz_release_complete AFTER DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
+    await q(`CREATE TRIGGER files_release_insert AFTER INSERT ON files REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION files_release_insert_statement()`)
+  }
+
+  async down() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`DROP TRIGGER IF EXISTS files_release_insert ON files`)
+    for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
+      await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
+    }
+    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+    await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
+    await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
+    await q(`DROP FUNCTION IF EXISTS files_release_insert_statement()`)
+    await q(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        UPDATE releases r SET complete = c.complete
+        FROM (SELECT release_complete(rid) AS complete) c
+        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
+      $$`)
+    await q(`DROP FUNCTION IF EXISTS recompute_release_complete(bigint)`)
+    await q(`DROP TABLE IF EXISTS release_dirty`)
+  }
+}

database/migrations/1768620765200_seed_range_priority.ts

@@ -0,0 +1,14 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** Seed ranges of a higher priority are leased first (`seed:plan --priority`), e.g. a gap in recent history. */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('seed_ranges', (table) => {
+      table.smallint('priority').notNullable().defaultTo(0)
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('seed_ranges', (table) => table.dropColumn('priority'))
+  }
+}

database/migrations/1768620765210_binary_part_count.ts

@@ -0,0 +1,21 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Stored parts per binary, kept by IngestService as it inserts parts
+ * (LOG.md 2026-10-06 (7)). claimCompleted counted every binary's parts in
+ * every batch (15% of Postgres' active time); a binary of thousands of
+ * parts that gets a few per batch was counted thousands of times. NULL:
+ * not counted yet (binaries from before), counted exactly on the next new
+ * part. Only parts of binaries without a file are deleted, so it stays exact
+ * until a binary completes.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
+    await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS part_count integer`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS part_count`)
+  }
+}

database/migrations/1768620765220_file_keys.ts

@@ -0,0 +1,69 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)).
+ * file_key is unique per files partition only (D39), so FileStore.insert
+ * looked every key up in every partition: 7 partitions and growing, 35,000
+ * random index reads for a batch of 5,000 files. With file_keys complete it
+ * asks file_keys first, and only keys whose hash is there (a duplicate, or
+ * rarely a collision) are looked up in files.
+ *
+ * New keys come from the files insert statement trigger. The keys from
+ * before are copied in the background (FileKeys.fill, gc:scheduler):
+ * copying them here took over half an hour of HDD reads and held up the
+ * deploy. Until file_keys_fill.complete, FileStore checks every partition.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`)
+    // One row: the fill's position in files.id, the id it must reach, and whether it got there.
+    await q(`
+      CREATE TABLE IF NOT EXISTS file_keys_fill (
+        id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1),
+        next_id bigint NOT NULL DEFAULT 0,
+        until_id bigint,
+        complete boolean NOT NULL DEFAULT false,
+        updated_at timestamptz NOT NULL DEFAULT now())`)
+    await q(`INSERT INTO file_keys_fill (id) VALUES (1) ON CONFLICT DO NOTHING`)
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          WITH added AS (
+            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
+                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
+                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
+            FROM (
+              SELECT release_id, size, name_extension(nm) AS e,
+                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
+              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
+            ) x
+            GROUP BY release_id
+          )
+          UPDATE releases r SET
+            file_count = r.file_count + a.n,
+            size = r.size + a.bytes,
+            extensions = CASE
+              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
+              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
+            search_files = CASE
+              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
+              ELSE nullif(left(concat_ws(' ', r.search_files, (
+                SELECT string_agg(w, ' ') FROM unnest(a.words) w
+                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
+          FROM added a
+          WHERE r.id = a.release_id;
+
+          INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
+          INSERT INTO file_keys (key_hash)
+            SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
+            ON CONFLICT DO NOTHING;
+          RETURN NULL;
+        END $$`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys_fill`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`)
+  }
+}

database/migrations/1768620765230_binaries_awaiting_inspection.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A completed binary whose file is queued for content inspection keeps its
+ * row until inspection has stored the file's fingerprint on it (LOG.md
+ * 2026-10-06 (11)): par2 naming (D16) matches par2 listings against binary
+ * fingerprints and attaches sets to binaries. With the inspection queue
+ * days behind, the GC deleted them first (after 6 hours unfingerprinted,
+ * 48 hours stale). TRUE while waiting; cleared by inspection.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
+    await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS awaiting_inspection boolean`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS awaiting_inspection`)
+  }
+}

database/migrations/1768620765240_binaries_gc_indexes.ts

@@ -0,0 +1,32 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Indexes for the GC's deletes of finished binaries, without the ones
+ * waiting for inspection (LOG.md 2026-10-06 (11)): those can be millions
+ * when inspection is behind, and the deletes must not read past them every
+ * sweep. Built CONCURRENTLY: binaries takes writes all the time.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_done_last_seen_idx ON binaries (last_seen_at)
+       WHERE file_id IS NOT NULL AND awaiting_inspection IS NOT TRUE`
+    )
+    await this.db.rawQuery(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_awaiting_inspection_idx ON binaries (last_seen_at)
+       WHERE awaiting_inspection IS TRUE`
+    )
+    await this.db.rawQuery(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS binaries_open_last_seen_idx ON binaries (last_seen_at)
+       WHERE completed_at IS NULL OR conflicted_at IS NOT NULL`
+    )
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_open_last_seen_idx`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_awaiting_inspection_idx`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS binaries_done_last_seen_idx`)
+  }
+}

database/migrations/1768620765250_release_completeness_narrower.ts

@@ -0,0 +1,94 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Release completeness (D24) for fewer releases, and without binaries
+ * (LOG.md 2026-10-07 (7)).
+ *
+ * - Only releases whose completeness can change are noted for the sweep: a
+ *   par2 listing, a file_total, or complete now (it can become incomplete).
+ *   For any other release release_complete() is false by definition; noting
+ *   every release a file went into made a sweep check 430,000 releases in
+ *   51 minutes.
+ * - release_complete() matches listed files by name against the files
+ *   themselves (coalesce(real_name, filename)); the size match still goes
+ *   through a binary when there is one. Seeding no longer creates binaries
+ *   for most files and the GC deletes them, so a name only reachable through
+ *   a binary was lost.
+ * - Notes that don't qualify are dropped.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`
+      CREATE OR REPLACE FUNCTION release_completeness_matters(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
+        SELECT EXISTS (SELECT 1 FROM releases r WHERE r.id = rid AND (r.file_total IS NOT NULL OR r.complete))
+            OR EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid)
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
+        INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL AND release_completeness_matters(rid)
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
+        WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
+        -- The release's files, once, by index (read only when there's a par2 listing to check).
+        have AS MATERIALIZED (
+          SELECT b.size, coalesce(f.real_name, f.filename) AS name
+          FROM files f LEFT JOIN binaries b ON b.file_id = f.id
+          WHERE f.release_id = rid)
+        SELECT coalesce((
+          SELECT CASE
+            WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
+            WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
+              SELECT 1 FROM release_files rf
+              WHERE rf.release_id = rid AND rf.size >= 65536
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
+                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.name = rf.name))
+            ELSE false END
+          FROM r), false)
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          WITH added AS (
+            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
+                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
+                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
+            FROM (
+              SELECT release_id, size, name_extension(nm) AS e,
+                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
+              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
+            ) x
+            GROUP BY release_id
+          )
+          UPDATE releases r SET
+            file_count = r.file_count + a.n,
+            size = r.size + a.bytes,
+            extensions = CASE
+              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
+              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
+            search_files = CASE
+              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
+              ELSE nullif(left(concat_ws(' ', r.search_files, (
+                SELECT string_agg(w, ' ') FROM unnest(a.words) w
+                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
+          FROM added a
+          WHERE r.id = a.release_id;
+
+          -- Only releases whose completeness can change (LOG.md 2026-10-07 (7)).
+          INSERT INTO release_dirty (release_id)
+            SELECT n.release_id FROM (SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL) n
+            WHERE release_completeness_matters(n.release_id);
+          INSERT INTO file_keys (key_hash)
+            SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
+            ON CONFLICT DO NOTHING;
+          RETURN NULL;
+        END $$`)
+    await q(`DELETE FROM release_dirty d WHERE NOT release_completeness_matters(d.release_id)`)
+  }
+
+  async down() {
+    // The previous definitions are in migrations …180, …190 and …220.
+    throw new Error('Not reversible: re-run the earlier definitions by hand if needed.')
+  }
+}

database/migrations/1768620765260_inspection_queue.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The content inspection backlog (LOG.md 2026-10-07 (8)): files waiting to be
+ * inspected, in Postgres instead of as BullMQ jobs. 5.4M waiting jobs took
+ * 3.3 GB of Redis memory and most of its single thread; InspectionQueue.feed
+ * keeps at most INSPECTION_FEED_MAX of them in BullMQ, where the workers and
+ * nodes take them. Lower priority first, then oldest.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS inspection_queue (
+        file_id bigint PRIMARY KEY,
+        priority smallint NOT NULL DEFAULT 0,
+        queued_at timestamptz NOT NULL DEFAULT now())`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS inspection_queue_order_idx ON inspection_queue (priority, queued_at)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS inspection_queue`)
+  }
+}

database/migrations/1768620765270_garbage_filter.ts

@@ -0,0 +1,64 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Dropping garbage uploads early (CONCEPT.md D42): random-named uploads that
+ * never get a real name or password.
+ * - upload_signatures: per posting tool (from the Message-ID) and first group,
+ *   how many probed random-named releases got a real name; `dropped` ones
+ *   aren't stored any more (GarbageFilter).
+ * - signature_probes: a sample of probed random-named releases, what
+ *   upload_signatures is learned from (the outcome is read from the release
+ *   at learning time, so names that come later count too).
+ * - dropped_releases: releases deleted after their probe, by release_key, so
+ *   seeding and ingest don't add them again.
+ * - is_real_release_name(): a name someone would search for. is_readable_name
+ *   lets spam through (usenetszene net, VTS_01_1.VOB, Nieuwe map, 100 MiB):
+ *   this also wants a year, episode, resolution, codec or language tag, or
+ *   three words (LOG.md 2026-10-08 (5)). No `?` in its regexes: knex takes it
+ *   for a binding (hence {0,1}).
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS upload_signatures (
+        tool varchar(32) NOT NULL,
+        group_name text NOT NULL,
+        probed integer NOT NULL DEFAULT 0,
+        named integer NOT NULL DEFAULT 0,
+        dropped boolean NOT NULL DEFAULT false,
+        updated_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (tool, group_name))`)
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS signature_probes (
+        release_id bigint PRIMARY KEY,
+        tool varchar(32) NOT NULL,
+        group_name text NOT NULL,
+        probed_at timestamptz NOT NULL DEFAULT now())`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS signature_probes_probed_at_idx ON signature_probes (probed_at)`)
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS dropped_releases (
+        release_key varchar(40) PRIMARY KEY,
+        reason varchar(16) NOT NULL,
+        dropped_at timestamptz NOT NULL DEFAULT now())`)
+    await this.db.rawQuery(`
+      CREATE OR REPLACE FUNCTION is_real_release_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+        SELECT name IS NOT NULL AND is_readable_name(name)
+          AND NOT (
+            name ~* '(usenet|video_ts|nieuwe map|new folder|(^|[^a-z0-9])mib([^a-z0-9]|$))'
+            OR name ~* '^(vts_|sample|cd[0-9])'
+            OR name ~ '^\\(.*\\)$'
+            OR name ~ '^[A-Za-z0-9]{8,}$')
+          AND (
+            name ~* '(^|[^a-z0-9])((19|20)[0-9]{2}|(480|576|720|1080|2160)[pi]|dl|nl)([^a-z0-9]|$)'
+            OR name ~* '(s[0-9]{2}e[0-9]{2}|x26[45]|h\\.{0,1}26[45]|hevc|web[- .]{0,1}(dl|rip)|bluray|remux|german|dutch|multi|flac|mp3|epub|pdf|ac3|dts|xxx)'
+            OR (SELECT count(*) FROM regexp_matches(name, '[A-Za-z]{3,}', 'g')) >= 3)
+      $$`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP FUNCTION IF EXISTS is_real_release_name(text)`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS dropped_releases`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS signature_probes`)
+    await this.db.rawQuery(`DROP TABLE IF EXISTS upload_signatures`)
+  }
+}

database/migrations/1768620765280_spot_listing_reads.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Listed spots (D29) were only read from their overview: no description, no
+ * NZB, so neither a password in the spot's text nor the NZB's own
+ * `<meta type="password">` was known (LOG.md 2026-10-08 (10)). nzb_read_at:
+ * when the spot's article and NZB were read for that (SpotService.readListing),
+ * also set while a reader holds it. The index finds the ones still to read,
+ * newest first.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS nzb_read_at timestamptz`)
+    await this.db.rawQuery(
+      `CREATE INDEX IF NOT EXISTS spots_listing_unread_idx ON spots (posted_at DESC) WHERE status = 'listed' AND nzb_read_at IS NULL`
+    )
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX IF EXISTS spots_listing_unread_idx`)
+    await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS nzb_read_at`)
+  }
+}

database/migrations/1768620765290_file_details.ts

@@ -0,0 +1,28 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * What inspection learns about a file (content type, archive listing) lives
+ * in file_details, on the NVMe, instead of in `files` (LOG.md 2026-10-08 (14)).
+ * `files` sits on the HDD and was updated after its insert by every probe:
+ * a new row version on a random page and an entry in each of its indexes
+ * (3.7M updates against 49.5M inserts in a day, 27% HOT). Its rows are now
+ * only inserted (and moved between releases or renamed, rarely).
+ * The existing values are copied by `files:move-details` after this deploy;
+ * a later migration drops the columns.
+ */
+export default class extends BaseSchema {
+  async up() {
+    // The foreign key locks `files` briefly: take the lock first, so it can't deadlock with writers.
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '60s'`)
+    await this.db.rawQuery(`LOCK TABLE files IN SHARE ROW EXCLUSIVE MODE`)
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS file_details (
+        file_id bigint PRIMARY KEY REFERENCES files (id) ON DELETE CASCADE,
+        content_type varchar(16),
+        archive jsonb)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS file_details`)
+  }
+}

database/schema.ts

@@ -33,14 +33,16 @@ export class AuthAccessTokenSchema extends BaseModel {
 }
 
 export class BinarySchema extends BaseModel {
-  static $columns = ['completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
+  static $columns = ['awaitingInspection', 'completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'partCount', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
   $columns = BinarySchema.$columns
+  @column()
+  declare awaitingInspection: boolean | null
   @column.dateTime()
   declare completedAt: DateTime | null
   @column.dateTime()
   declare conflictedAt: DateTime | null
   @column()
-  declare fileId: number | null
+  declare fileId: bigint | number | null
   @column()
   declare fileKey: string
   @column.dateTime()
@@ -57,6 +59,8 @@ export class BinarySchema extends BaseModel {
   declare name: string
   @column()
   declare parentBinaryId: bigint | number | null
+  @column()
+  declare partCount: number | null
   @column.dateTime()
   declare postedAt: DateTime | null
   @column()
@@ -147,9 +151,22 @@ export class DeferredRangeSchema extends BaseModel {
   declare lastArticle: bigint | number
 }
 
+export class DroppedReleaseSchema extends BaseModel {
+  static $columns = ['droppedAt', 'reason', 'releaseKey'] as const
+  $columns = DroppedReleaseSchema.$columns
+  @column.dateTime()
+  declare droppedAt: DateTime
+  @column()
+  declare reason: string
+  @column({ isPrimary: true })
+  declare releaseKey: string
+}
+
 export class FetchRangeSchema extends BaseModel {
-  static $columns = ['createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
+  static $columns = ['backfill', 'createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
   $columns = FetchRangeSchema.$columns
+  @column()
+  declare backfill: boolean
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
   @column()
@@ -170,8 +187,41 @@ export class FetchRangeSchema extends BaseModel {
   declare updatedAt: DateTime
 }
 
+export class FileDetailSchema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'fileId'] as const
+  $columns = FileDetailSchema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column({ isPrimary: true })
+  declare fileId: bigint | number
+}
+
+export class FileKeySchema extends BaseModel {
+  static $columns = ['keyHash'] as const
+  $columns = FileKeySchema.$columns
+  @column({ isPrimary: true })
+  declare keyHash: bigint | number
+}
+
+export class FileKeysFillSchema extends BaseModel {
+  static $columns = ['complete', 'id', 'nextId', 'untilId', 'updatedAt'] as const
+  $columns = FileKeysFillSchema.$columns
+  @column()
+  declare complete: boolean
+  @column({ isPrimary: true })
+  declare id: number
+  @column()
+  declare nextId: bigint | number
+  @column()
+  declare untilId: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime
+}
+
 export class FileSchema extends BaseModel {
-  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'groups', 'id', 'messageIds', 'parts', 'poster', 'realName', 'releaseId', 'size', 'updatedAt'] as const
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
   $columns = FileSchema.$columns
   @column()
   declare archive: any | null
@@ -186,11 +236,11 @@ export class FileSchema extends BaseModel {
   @column()
   declare filename: string
   @column()
+  declare firstMessageId: string | null
+  @column()
   declare groups: any | null
   @column({ isPrimary: true })
-  declare id: number
-  @column()
-  declare messageIds: any
+  declare id: bigint | number
   @column()
   declare parts: number
   @column()
@@ -200,16 +250,133 @@ export class FileSchema extends BaseModel {
   @column()
   declare releaseId: bigint | number | null
   @column()
+  declare segments: Buffer
+  @column()
+  declare size: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime | null
+}
+
+export class FilesP0Schema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
+  $columns = FilesP0Schema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime | null
+  @column()
+  declare date: bigint | number
+  @column()
+  declare fileKey: string | null
+  @column()
+  declare filename: string
+  @column()
+  declare firstMessageId: string | null
+  @column()
+  declare groups: any | null
+  @column({ isPrimary: true })
+  declare id: bigint | number
+  @column()
+  declare parts: number
+  @column()
+  declare poster: string
+  @column()
+  declare realName: string | null
+  @column()
+  declare releaseId: bigint | number | null
+  @column()
+  declare segments: Buffer
+  @column()
+  declare size: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime | null
+}
+
+export class FilesP1000000Schema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
+  $columns = FilesP1000000Schema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime | null
+  @column()
+  declare date: bigint | number
+  @column()
+  declare fileKey: string | null
+  @column()
+  declare filename: string
+  @column()
+  declare firstMessageId: string | null
+  @column()
+  declare groups: any | null
+  @column({ isPrimary: true })
+  declare id: bigint | number
+  @column()
+  declare parts: number
+  @column()
+  declare poster: string
+  @column()
+  declare realName: string | null
+  @column()
+  declare releaseId: bigint | number | null
+  @column()
+  declare segments: Buffer
+  @column()
+  declare size: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime | null
+}
+
+export class FilesP5000000Schema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
+  $columns = FilesP5000000Schema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime | null
+  @column()
+  declare date: bigint | number
+  @column()
+  declare fileKey: string | null
+  @column()
+  declare filename: string
+  @column()
+  declare firstMessageId: string | null
+  @column()
+  declare groups: any | null
+  @column({ isPrimary: true })
+  declare id: bigint | number
+  @column()
+  declare parts: number
+  @column()
+  declare poster: string
+  @column()
+  declare realName: string | null
+  @column()
+  declare releaseId: bigint | number | null
+  @column()
+  declare segments: Buffer
+  @column()
   declare size: bigint | number | null
   @column.dateTime({ autoCreate: true, autoUpdate: true })
   declare updatedAt: DateTime | null
 }
 
 export class GroupSchema extends BaseModel {
-  static $columns = ['active', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
+  static $columns = ['active', 'backfillNextId', 'backfillUntilId', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
   $columns = GroupSchema.$columns
   @column()
   declare active: boolean
+  @column()
+  declare backfillNextId: bigint | number | null
+  @column()
+  declare backfillUntilId: bigint | number | null
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime | null
   @column()
@@ -226,6 +393,17 @@ export class GroupSchema extends BaseModel {
   declare updatedAt: DateTime | null
 }
 
+export class InspectionQueueSchema extends BaseModel {
+  static $columns = ['fileId', 'priority', 'queuedAt'] as const
+  $columns = InspectionQueueSchema.$columns
+  @column({ isPrimary: true })
+  declare fileId: bigint | number
+  @column()
+  declare priority: number
+  @column.dateTime()
+  declare queuedAt: DateTime
+}
+
 export class PartConflictSchema extends BaseModel {
   static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const
   $columns = PartConflictSchema.$columns
@@ -272,6 +450,13 @@ export class PartSchema extends BaseModel {
   declare pcrc32: string | null
 }
 
+export class ReleaseDirtySchema extends BaseModel {
+  static $columns = ['releaseId'] as const
+  $columns = ReleaseDirtySchema.$columns
+  @column()
+  declare releaseId: bigint | number
+}
+
 export class ReleaseFileSchema extends BaseModel {
   static $columns = ['md5', 'md516K', 'name', 'par2FileId', 'releaseId', 'setId', 'size'] as const
   $columns = ReleaseFileSchema.$columns
@@ -292,7 +477,7 @@ export class ReleaseFileSchema extends BaseModel {
 }
 
 export class ReleaseSchema extends BaseModel {
-  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'postedAt', 'poster', 'releaseKey', 'size', 'source'] as const
+  static $columns = ['category', 'complete', 'container', 'contents', 'contentsWalkedAt', 'encrypted', 'extensions', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const
   $columns = ReleaseSchema.$columns
   @column()
   declare category: number | null
@@ -302,9 +487,13 @@ export class ReleaseSchema extends BaseModel {
   declare container: string | null
   @column()
   declare contents: any | null
+  @column.dateTime()
+  declare contentsWalkedAt: DateTime | null
   @column()
   declare encrypted: boolean | null
   @column()
+  declare extensions: any | null
+  @column()
   declare fileCount: number
   @column()
   declare fileTotal: number | null
@@ -320,6 +509,8 @@ export class ReleaseSchema extends BaseModel {
   declare name: string
   @column({ serializeAs: null })
   declare password: string | null
+  @column()
+  declare passwordSource: string | null
   @column.dateTime()
   declare postedAt: DateTime | null
   @column()
@@ -327,13 +518,101 @@ export class ReleaseSchema extends BaseModel {
   @column()
   declare releaseKey: string
   @column()
+  declare searchFiles: string | null
+  @column()
+  declare searchText: string | null
+  @column()
   declare size: bigint | number
   @column()
   declare source: string | null
 }
 
+export class SeedNodeSchema extends BaseModel {
+  static $columns = ['createdAt', 'disabled', 'id', 'lastSeenAt', 'name', 'tokenHash'] as const
+  $columns = SeedNodeSchema.$columns
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare disabled: boolean
+  @column({ isPrimary: true })
+  declare id: number
+  @column.dateTime()
+  declare lastSeenAt: DateTime | null
+  @column()
+  declare name: string
+  @column()
+  declare tokenHash: string
+}
+
+export class SeedRangeSchema extends BaseModel {
+  static $columns = ['attempts', 'createdAt', 'doneAt', 'endId', 'error', 'files', 'fragments', 'groupName', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'startId', 'status'] as const
+  $columns = SeedRangeSchema.$columns
+  @column()
+  declare attempts: number
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column.dateTime()
+  declare doneAt: DateTime | null
+  @column()
+  declare endId: bigint | number
+  @column()
+  declare error: string | null
+  @column()
+  declare files: bigint | number
+  @column()
+  declare fragments: bigint | number
+  @column()
+  declare groupName: string
+  @column()
+  declare headers: bigint | number
+  @column.dateTime()
+  declare heartbeatAt: DateTime | null
+  @column({ isPrimary: true })
+  declare id: bigint | number
+  @column.dateTime()
+  declare leasedAt: DateTime | null
+  @column()
+  declare nextId: bigint | number
+  @column()
+  declare nodeId: number | null
+  @column()
+  declare priority: number
+  @column()
+  declare startId: bigint | number
+  @column()
+  declare status: string
+}
+
+export class SfvEntrySchema extends BaseModel {
+  static $columns = ['crc', 'name', 'postedAt', 'poster', 'releaseId'] as const
+  $columns = SfvEntrySchema.$columns
+  @column()
+  declare crc: string
+  @column()
+  declare name: string
+  @column.dateTime()
+  declare postedAt: DateTime | null
+  @column()
+  declare poster: string | null
+  @column({ isPrimary: true })
+  declare releaseId: bigint | number
+}
+
+export class SignatureProbeSchema extends BaseModel {
+  static $columns = ['groupName', 'probedAt', 'releaseId', 'tool'] as const
+  $columns = SignatureProbeSchema.$columns
+  @column()
+  declare groupName: string
+  @column.dateTime()
+  declare probedAt: DateTime
+  @column({ isPrimary: true })
+  declare releaseId: bigint | number
+  @column()
+  declare tool: string
+}
+
 export class SpotSchema extends BaseModel {
-  static $columns = ['articleNumber', 'category', 'createdAt', 'error', 'groupName', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'website'] as const
+  static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'nzbReadAt', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const
   $columns = SpotSchema.$columns
   @column()
   declare articleNumber: bigint | number | null
@@ -342,14 +621,22 @@ export class SpotSchema extends BaseModel {
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
   @column()
+  declare description: string | null
+  @column.dateTime()
+  declare detailsAt: DateTime | null
+  @column()
   declare error: string | null
   @column()
   declare groupName: string
   @column()
+  declare image: any | null
+  @column()
   declare keyId: number | null
   @column({ isPrimary: true })
   declare messageId: string
   @column.dateTime()
+  declare nzbReadAt: DateTime | null
+  @column.dateTime()
   declare postedAt: DateTime | null
   @column()
   declare poster: string
@@ -370,9 +657,41 @@ export class SpotSchema extends BaseModel {
   @column()
   declare title: string
   @column()
+  declare verified: boolean | null
+  @column()
   declare website: string | null
 }
 
+export class StagedSegmentSchema extends BaseModel {
+  static $columns = ['counterKey', 'fileKey', 'segments', 'stagedAt'] as const
+  $columns = StagedSegmentSchema.$columns
+  @column()
+  declare counterKey: string | null
+  @column({ isPrimary: true })
+  declare fileKey: string
+  @column()
+  declare segments: any
+  @column.dateTime()
+  declare stagedAt: DateTime
+}
+
+export class UploadSignatureSchema extends BaseModel {
+  static $columns = ['dropped', 'groupName', 'named', 'probed', 'tool', 'updatedAt'] as const
+  $columns = UploadSignatureSchema.$columns
+  @column()
+  declare dropped: boolean
+  @column()
+  declare groupName: string
+  @column()
+  declare named: number
+  @column()
+  declare probed: number
+  @column({ isPrimary: true })
+  declare tool: string
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime
+}
+
 export class UserSchema extends BaseModel {
   static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
   $columns = UserSchema.$columns

lib/aggregate/index.ts

@@ -0,0 +1,86 @@
+/**
+ * Edge aggregation (CONCEPT.md D37): groups header segments into files in
+ * memory, so a seeding node writes one row per file instead of one per
+ * article. A file is done once every part number from 1 to its total has
+ * an article; what never completes within the node's window (an upload cut
+ * by the range edges, a random subject per article) leaves as fragments
+ * for the regular ingest path, where it can meet its other parts.
+ */
+
+/** What the aggregator needs of a segment (IngestSegment has all of it). */
+export interface AggregatableSegment {
+  fileKey: string
+  number: number
+  totalParts: number
+  messageId: string
+  bytes: number
+}
+
+export interface AggregatedFile<S extends AggregatableSegment> {
+  fileKey: string
+  /** One segment per part number, in part order; the first article seen wins (a later one is a repost). */
+  segments: S[]
+}
+
+interface OpenFile<S> {
+  parts: Map<number, S>
+  totalParts: number
+  /** Batch in which a segment last arrived. */
+  touched: number
+}
+
+export class FileAggregator<S extends AggregatableSegment> {
+  private open = new Map<string, OpenFile<S>>()
+  private batch = 0
+  /** Segments with a part number outside 1..total: never part of a complete file. */
+  private stray: S[] = []
+
+  /** Adds one batch of segments; returns the files they completed. */
+  add(segments: S[]): AggregatedFile<S>[] {
+    this.batch++
+    const touched = new Set<string>()
+    for (const segment of segments) {
+      if (segment.number < 1 || segment.number > segment.totalParts) {
+        this.stray.push(segment)
+        continue
+      }
+      let file = this.open.get(segment.fileKey)
+      if (!file) {
+        file = { parts: new Map(), totalParts: segment.totalParts, touched: this.batch }
+        this.open.set(segment.fileKey, file)
+      }
+      if (!file.parts.has(segment.number)) file.parts.set(segment.number, segment)
+      file.touched = this.batch
+      touched.add(segment.fileKey)
+    }
+
+    const complete: AggregatedFile<S>[] = []
+    for (const key of touched) {
+      const file = this.open.get(key)!
+      if (file.parts.size < file.totalParts) continue
+      this.open.delete(key)
+      complete.push({ fileKey: key, segments: [...file.parts.values()].sort((a, b) => a.number - b.number) })
+    }
+    return complete
+  }
+
+  /**
+   * Gives up on files that got no segment in the last `idleBatches` batches
+   * (all open files when 0) and returns their segments, plus stray ones.
+   */
+  evict(idleBatches = 0): S[] {
+    const out = this.stray
+    this.stray = []
+    for (const [key, file] of this.open) {
+      if (idleBatches > 0 && this.batch - file.touched < idleBatches) continue
+      this.open.delete(key)
+      out.push(...file.parts.values())
+    }
+    return out
+  }
+
+  /** Files still waiting for parts. */
+  get openFiles(): number {
+    return this.open.size
+  }
+}