REBUILD.md: a pending release without a target is exhausted at once

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

.adonisjs/server/controllers.ts

@@ -6,6 +6,7 @@
 export const controllers = {
   AccessTokens: () => import('#controllers/access_tokens_controller'),
   NewAccount: () => import('#controllers/new_account_controller'),
+  Newznab: () => import('#controllers/newznab_controller'),
   Node: () => import('#controllers/node_controller'),
   Profile: () => import('#controllers/profile_controller'),
   Releases: () => import('#controllers/releases_controller'),

.adonisjs/server/routes.d.ts

@@ -4,29 +4,39 @@ type ParamValue = string | number | bigint | boolean
 
 export type ScannedRoutes = {
   ALL: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'auth.new_account.store': { paramsTuple?: []; params?: {} }
     'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.lease': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   GET: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   HEAD: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   POST: {
@@ -38,6 +48,7 @@ export type ScannedRoutes = {
     'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
 }
 declare module '@adonisjs/core/types/http' {

.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+build
+tmp
+.env
+.git
+edge/target

.gitea/workflows/deploy.yaml

@@ -1,9 +1,12 @@
-# Tests every push; a push to master that passes is deployed to homelab.
+# Tests every push; a push to rebuild (voyager 2) that passes is deployed to
+# homelab.
 #
 # Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
 # run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
-# which fetches the commit from this repo, builds the image, runs the
-# migrations and recreates the app services (never postgres or redis).
+# which passes "deploy-rebuild <sha>" on to ~/bin/deploy-usenet-rebuild
+# (deploy/deploy.sh): it fetches the commit from this repo, builds the
+# images, runs the migrations and recreates the app services (never
+# postgres). See deploy/README.md.
 name: Test and deploy
 
 on:
@@ -56,16 +59,29 @@ jobs:
           node-version-file: .nvmrc
           cache: npm
       - run: npm ci
+      - name: Rust toolchain
+        run: |
+          command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+      # The edge worker (edge/, Rust): built first, since the e2e suite runs edge/target/release/edge-node against
+      # the node API; then its own tests, and the Rust and TypeScript RangeIndexers compared on the committed
+      # captures (edge/fixtures).
+      - name: Build the edge worker
+        run: cargo build --release --manifest-path edge/Cargo.toml --bin edge-node
       - name: Type check
         run: npx tsc --noEmit
       - name: Migrate
         run: node ace migration:run --force
       - name: Test
         run: node ace test
+      - name: cargo test
+        run: cargo test --release --manifest-path edge/Cargo.toml
+      - name: The Rust indexer over the fixtures
+        run: edge/bench-fixtures.sh
 
   deploy:
-    needs: test
-    if: github.ref == 'refs/heads/master'
+    needs: [test]
+    if: github.ref == 'refs/heads/rebuild'
     runs-on: ubuntu-latest
     steps:
       - name: Deploy to homelab
@@ -78,4 +94,4 @@ jobs:
           chmod 600 ~/.ssh/deploy_key
           printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
           ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
-            daan@10.206.0.175 "deploy ${{ github.sha }}"
+            daan@10.206.0.175 "deploy-rebuild ${{ github.sha }}"

.gitignore

@@ -24,3 +24,6 @@ yarn-error.log
 
 # Platform specific
 .DS_Store
+
+# Generated by the dev tooling
+.adonisjs/server/routes.json

Dockerfile

@@ -0,0 +1,45 @@
+# syntax=docker/dockerfile:1
+#
+# Voyager 2's images (deploy/compose.yaml):
+#   --target app   the AdonisJS app: web server, coordinator, migrations, ace commands
+#   --target edge  the Rust edge worker (edge/, bin edge-node)
+
+# Node is pinned as in CI (.nvmrc): on 24.21, jsonschema 1.5.0 (used by Ace to
+# validate commands) throws "Invalid URL" on $ref, so no ace command starts.
+ARG NODE_VERSION=24.16
+
+# The Rust worker (before the app: its image carries edge-bench). Registry and target dir are build caches, so a rebuild only compiles what changed.
+FROM rust:1-bookworm AS edge-build
+WORKDIR /src
+COPY edge ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/src/target \
+    cargo build --release --locked --bin edge-node --bin edge-bench \
+ && cp target/release/edge-node target/release/edge-bench /usr/local/bin/
+
+# The app, compiled, with production dependencies only.
+FROM node:${NODE_VERSION}-bookworm AS app-build
+WORKDIR /src
+COPY package.json package-lock.json ./
+RUN npm ci
+COPY . .
+RUN node ace build \
+ && cd build \
+ && npm ci --omit=dev
+
+FROM node:${NODE_VERSION}-bookworm-slim AS app
+ENV NODE_ENV=production
+WORKDIR /app
+COPY --from=app-build /src/build ./
+# The Rust indexer's bench: `node ace replay:corpus` runs a group's corpus through it (the replay harness).
+COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
+USER node
+# Every service picks its own command, e.g. `node ace coordinator:run`.
+CMD ["node", "ace", "list"]
+
+FROM debian:bookworm-slim AS edge
+COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node
+# edge-bench replays the corpus for the grouping sweeps (deploy/corpus_sweep.py).
+COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
+USER 1000:1000
+ENTRYPOINT ["/usr/local/bin/edge-node"]

REBUILD.md

@@ -1,6 +1,12 @@
-# Rebuild: header-first indexer (spec, draft for approval)
+# Voyager 2: header-first indexer (concept)
 
-Branch `rebuild`. Phase 1 finds releases from headers and makes their NZBs.
+The concept for what runs (since 2026-10-11; it started as the spec of a
+rebuild, then a draft for approval). Voyager 1's CONCEPT.md and its
+decisions D1–D42 are superseded; its domain model still holds, and its
+features not yet here are the phase 2 backlog below.
+
+Branch `rebuild`; its successor is designed under *Voyager 3* below (branch
+`voyager-3`). Phase 1 finds releases from headers and makes their NZBs.
 Everything learned from articles (names from yEnc headers, par2, archive
 listings, encryption, spots) is phase 2 and adds to what phase 1 stored; it
 never rewrites it. Sources: CONCEPT.md, LOG.md 2026-10-08 (13)–(14).
@@ -54,8 +60,33 @@ most workers on batcave, a few on poweredge-kantoor with
 `edge:node --max-mbit` (e.g. 50 of its 160 Mbit/s ≈ 100–125 k articles/s;
 the cap is shared by the node's processes).
 
+Every group is tiled from article 1 (2026-10-28): one "ancient" range,
+status `ancient`, covers 1 … low watermark − 1, what had expired on the
+server before the group was planned. Never leased, fetched or filled —
+nothing selects it — and the status panel reports it apart ("Ancient: …
+articles below the low watermark"); migration 6700 added it for the
+groups planned before, `Planner.historyRows` for groups planned since.
+
+Also built (2026-10-09): the live tail (new articles planned every 2
+minutes from one LIST ACTIVE per name prefix, at priority −1: leased and
+fetched before history; since 2026-10-27 a round grows the group's open
+tail range, a planned one at the tail's priority nothing spooled or
+took, up to RANGE_ARTICLES before it adds rows, as a tail nothing leased
+had become a sliver a group a round, 127k rows of a median 14k articles;
+`ranges:merge-tail` packs what such rounds left); the catalog API,
+Newznab (`/api`) and the web
+interface, from voyager 1 (the current build), with categories from the
+name; voyager 1's 29 learned garbage signatures (`rules:signatures`,
+`database/data/voyager1_signatures.tsv`).
+
+Deployment (2026-10-10): `Dockerfile` (targets app and edge),
+`deploy/compose.yaml` (project usenet-rebuild on homelab), `deploy/deploy.sh`;
+a push to `rebuild` that passes CI is deployed. See `deploy/README.md`.
+
 Not built yet: re-running ranges for a retracted rule (the counts and rule
-versions are recorded), deployment (Dockerfile, compose), phase 2.
+versions are recorded), decision f, phase
+2 (inspection of the probe queue: yEnc names, encryption, archives, par2;
+passwords; spots).
 
 ## Target
 
@@ -99,15 +130,30 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
    settles).
 
 2. **Edge worker** (node, TypeScript reusing `lib/`). Measured (`edge:bench`,
-   2026-10-09): 73 k headers/s per core on an a.b.ath range from 2013,
-   103 k/s on a recent a.b.boneless range (2 M headers each), ~1.2 GB peak
-   per process; the 25 B lines/day goal needs ~3–4 cores over all nodes,
-   so no Rust. Per lease: compressed chunks of 10–50 k lines, in order →
+   2026-10-09, after two rounds of work): ~200–215 k headers/s per core on
+   recent a.b.boneless, ~320–340 k on a 2013 a.b.ath range, ~0.7 GB peak
+   per process. **Rust** (`edge/`, the same RangeIndexer path
+   over the same captures; `check.ts` finds every output identical, the
+   decoded payloads included): 3.5–3.8× less CPU (boneless 2.9 vs 11.1
+   CPU-s per 2 M headers, ath 2.15 vs 7.5) and a third of the memory
+   (0.20–0.24 GB). Its gains: no GC, hand-written matching instead of the
+   regexes, mimalloc, libdeflate (half of zlib's inflate time) and SIMD line
+   scanning. At the full target (~1.25 M/s) that is ~1.5–2 worker cores
+   instead of ~6–7; the speed is still set by the NNTP connections. The
+   cost: a second implementation of lib/headers, lib/release, lib/rules and
+   the chunk format, kept identical (check.ts) for as long as both exist.
+   Findings, how to run both and how to compare their outputs:
+   `edge/README.md`. Since 2026-10-09 the Rust worker
+   (`edge-node`, same crate) is the one running on homelab: ~8× less CPU
+   per header and 20× less memory than `edge:node` in production.
+   Per lease: compressed chunks of 10–50 k lines, in order →
    parse subjects → skip crossposts outside their owner group (of the
    indexed groups in the article's Xref, the one with the lowest priority
    number; see the group list) → group articles into files → files
    into releases (`lib/release` rules) → drop garbage (random name +
-   dropped signature or tool) → set aside per-article random subjects and
+   dropped signature or tool), bare executables, dropped posters, and the
+   single-archive fakes (stub archives and nonsense pieces, below) → set
+   aside per-article random subjects and
    obfuscated subjects (counted only; phase 2). Returns per batch: release
    chunks, edge fragments, probe candidates, counters. Holds no state beyond
    its lease.
@@ -158,13 +204,22 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
    number; merged when the neighbouring range is done, then written as a
    chunk. Never staged per segment.
 
-8. **Settling**. A release is settled when the ranges covering its posting
-   span are done and no chunk arrived for a day: completeness is computed
-   then (expected vs found files and segments). Phase 2 picks its work from
-   settled releases.
+8. **Settling**. A release is settled when nothing was added to it for 6
+   hours (30 minutes once its last post is that old: history), and its
+   completeness decided: complete when every file its [i/N] counter
+   announced was found. Without a counter, by its par2 sets once their index
+   is probed (`par2_completeness`): every file a set describes is there by
+   name or by its first 16 KiB, or, renamed, as many files and bytes; decided
+   again after a par2 merge and before the pruner drops a set's file list.
+   Segments aren't compared. Phase 2 picks its work from settled releases.
 
-9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
-   search (names) and get. Web UI later.
+9. **NZB and API**. NZB built from a release's chunks, with a known
+   password in its meta; Newznab search (names, inner names) and get; a web
+   interface. Categories (Newznab's) by a trigger on releases (migration
+   5200): from the name, else the inner name the prober found, else a spot's
+   category (spot_category, set by the spot inspector), else the groups
+   (erotica, sounds, ebooks, …). `releases:recategorize` recomputes old rows
+   in batches.
 
 ## Group list
 
@@ -177,11 +232,656 @@ of other groups whose counts show crossposts skipped for it (part 5). Start:
 the 1000 largest `alt.binaries.*` groups by LIST ACTIVE (~266 k ranges of
 2 M articles; boneless alone 80 k).
 
-## Not in phase 1
+## Corpus
 
-Inspection and archive walks, par2/SFV naming, spots, NZB imports, the
-after-probe garbage deletes, live indexing (the current indexer keeps
-running and does those meanwhile).
+The grouping sweeps' dataset (2026-10-12; LOG.md 2026-10-12 (12)): the XOVER
+chunks of ranges marked `capture`, kept on the HDD in
+`corpus/<group>/<start>-<end>/` (edge-bench's layout, `range.json` once
+complete), up to 300 GB. The prefetcher stores them as it fetches and reads
+them from there afterwards, so a reset re-indexes them without the news
+server. `deploy/corpus_sweep.py` marks more ranges each sweep (`capture`,
+spread over groups and eras, the top-100 groups first), replays every stored
+range with the deployed build's edge-bench (`replay`, ~seconds a range) and
+reports suspects and the change since the previous build (`report`). A rule
+change is judged on the whole corpus before a reset.
+
+Since 2026-10-23 also history fetched ahead of the indexing: while the
+catalog is over its NVMe cap, ranges marked `capture` (priority >= 0, not
+leased) are fetched into the corpus by idle loops, at most perGroup /
+perRange of a group at a time, to be indexed from disk once there's room
+(LOG.md 2026-10-23 (100)). Stored filtered (`CORPUS_FILTER`, default on):
+without the lines the edge worker only counts, decided per line as it does
+(a crosspost another indexed group owns, a subject that names no file part;
+~48% of headers), the chunk compressed again as zlib; beside it a `.drops`
+(gzip JSON: rule version, the owners' priorities, per rule the article
+numbers as differences, crossposts by owner) and `filtered` in range.json.
+The worker makes the same releases of a filtered chunk; rule_counts lack
+those two rules' counts. `corpus:filter` filters chunks stored raw. A
+corpus filtered this way no longer serves a sweep of those two rules.
+
+Windows by posting time (2026-10-25; voyager 3's test data, LOG.md (104)):
+`ranges:dates` samples when each active group's articles were posted (one
+XOVER of 32 articles every 500M articles of the group, 16 … 1000 samples,
+the median Date of each slice, kept monotone; `group_dates`; over plain
+connections: Eweka cuts a compressed reply at the first ".\r\n" in its
+output, which some small replies begin with, `TruncatedReply`, LOG.md
+2026-10-25 (105)) and estimates
+every range's posting date from the curve (`ranges.est_date`; the edges of
+a window are off by up to one sample interval, three weeks in boneless).
+`ranges:capture --from 2025-01-01 --to 2026-01-01` marks every active
+group's ranges posted in the window `capture`: planned ones not started at
+`--priority` (0), done ones as they are, both fetched into the corpus by the
+fill, which takes capture ranges in priority order, done ones too (they're
+fetched again, corpus only), and skips those complete (`ranges.stored_at`,
+set when it sees the range.json). A year of every group is the data to
+replay voyager 3's 30-day blocks on: marked 2026-10-25, 2016 (priority 0,
+30B articles, ~0.56 TB) and 2021 (priority 1, 87B, ~1.6 TB); a recent year
+is ~200B articles, ~3.6 TB.
+
+## Phase 2
+
+Built: the prober (2026-10-11; `probe:run`, LOG.md 2026-10-11 (2)). It
+reads the probe queue's first articles (`lone`, `par2`, `archive`) and
+stores per release whether it's passworded and its inner name, which search,
+the web interface and Newznab use.
+
+Built: the par2 merge (2026-10-12; `releases:merge`, the `merger` service,
+LOG.md 2026-10-12 (13)). The prober stores per probed file start the MD5 of
+its first 16 KiB (`file_hashes`) and per par2 body its set's file list
+(`par2_files`, `par2_sets`), which holds the same hash per file whatever the
+file was renamed to. Every minute the merger merges the releases of one
+group, posted within 48 hours of a release holding the set's par2, whose
+files the set describes, into the one with the most files (chunks,
+candidates, inspections, counts; the par2's shared name when none was
+readable); `release_merges` keeps what went where. Obfuscated uploads whose
+files were each a release of their own (`lone` candidates) become one.
+Readable files have no hash: a name pass (2026-10-23) joins the releases
+of the set's poster in its group posted within 12 hours whose files the
+set all names (no case or punctuation; sidecars, par2 and images aside),
+as `via = 'name'`: an album's tracks and its index, an index beside its
+differently spelled video. `releases:merge-names` for sets from before.
+Also merged: lone files whose inner names share a random stem (`inner`),
+and small sidecar-only releases into their upload (`sidecar`).
+It reads what's new an hour (`created_at`) at a time, its place kept after
+each, and merges only the sets held or matched by two releases or more; a
+file whose first 16 KiB and size more than 20 files share is a common one and
+matches nothing (LOG.md 2026-10-20 (85)).
+A set's file list is dropped once it can't merge anything more: complete (all
+its files in its holder), judged (every holder junk), or final (every holder
+inside a run of done ranges, those ranges' candidates probed, its hashes seen
+by the merger); `par2_sets` stays (LOG.md 2026-10-20 (87)).
+
+**Backlog: voyager 1's features not in voyager 2** (CONCEPT.md decisions;
+voyager 1's data exported to `/mnt/bulk/usenet-rebuild/v1-export/` on
+homelab: `voyager1-tables.dump` with 2.76 M spots, upload signatures,
+signature probes, dropped releases and groups; `voyager1-release-passwords.csv`
+with 68,032 releases with a password):
+
+| Feature | Voyager 1 | In voyager 2 |
+|---|---|---|
+| Spotnet spots | D29 | the spotter and the spot inspector (below); new spots only, the export's not imported |
+| Posted NZBs read into releases (also NZBs inside archives) | D26, D28 | not yet |
+| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
+| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
+| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
+| 7z listings | D19 | yes (2026-10-23): the prober reads the `7z-tail` (the last segment of the highest volume, or of an unsplit multi-segment .7z) and `lib/sevenzip` lists it from the archive's end header (LZMA-packed, decoded in TypeScript); an encrypted header (`-mhe`) means passworded without names. One release keeps only the tails of its highest volume (`seven_zip_tails`). ZIP: the first segment's local headers (first entries, encryption) |
+| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
+| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
+
+Live indexing (the tail) runs since 2026-10-09.
+
+**Junk** (voyager 1's second filter, D42). After the header pass's rules
+(garbage signatures, executables, per-article and obfuscated subjects), the
+judge (`releases:judge`, service `judge`) judges releases by what the
+prober, the merges and the spots found: every 2 minutes the releases whose
+facts changed (noted by triggers in `judge_dirty`), daily every release (for
+what time changes: a day after settling, ranges read beside it). In this
+order (the first rule wins): *dead* (every probed first article gone), *fake* (a film, episode
+or music name whose content is a program: archive entries all
+executables, or the file itself one by name or by its first bytes, sniffed
+`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
+*locked* (passworded, no password from a spot, its NZB or the title),
+*tiny* (one file of one article, 2–100 kB, under a readable release-like
+name outside the music, e-book and comic groups, without an e-book,
+subtitle, crack or sidecar word: the catalog's side of the edge's
+tiny-archive rule, 1.5M rows of the 2015–17 flood that the edge indexed
+before the rule; LOG.md (131), (135)), *unusable* (random names, no inner
+name, no par2 or inner-name merge, no spot, a day after settling).
+Verdicts (`release_verdicts`, with the rule
+set's version) hide releases from search (`judged` shows them); a verdict
+whose rule no longer matches is taken back. Nothing is dropped yet.
+
+**Sidecars** (the merger's third pass). An upload's .nfo, .sfv or .nzb
+that came apart from it (another name, another counter: "[00/80] … .nzb"
+beside "[01/94] … .part01.rar") was a release of its own. The settler
+queues small settled releases (≤ 3 files, < 1 MB) in `fold_candidates`;
+one whose files are all sidecars joins the poster's bigger release in the
+same group posted within 10 minutes: the one sharing a word of 4+ letters
+with its subject, else the one under the same counter total, else none
+(`release_merges.via` 'sidecar').
+
+**Spots** (after voyager 1's D29; `lib/spotnet` ported as is). Two
+processes, one NNTP connection each:
+
+- **The spotter** (`spots:read`, service `spotter`) reads free.pt's
+  overviews after the last article it saw (`spot_state`; the first time the
+  last `SPOT_BACKFILL`, 20,000, ~3 months), every `SPOT_EVERY_SECONDS`
+  (120). For each spot it reads the full header: the signature is checked
+  there as Spotweb does (Spotnet's keys 2–4; self-signed key 7 by hashcash),
+  and the X-XML gives the description, the website and the NZB's articles.
+  A password the title (`{{pw}}`) or the description names
+  (`passwordFromSpot`) is kept with the spot, with its source. A
+  delete/dispose message marks its target `removed` (a personal one only
+  from the same spotter within 5 days) and takes its links and passwords
+  back from the releases.
+- **The spot inspector** (`spots:inspect`, service `spot-inspector`) looks
+  inside each listed spot, newest first: it reads the compressed NZB (≤
+  `SPOT_MAX_NZB_SEGMENTS` articles in alt.binaries.ftd), lists its files
+  in `spot_files` (named as the header pass names them, `parseSubject`),
+  and matches each file to the release the header pass built for it: the
+  same release identity (`releaseIdentity` over the file's subject and
+  poster) in one of its groups, the release posted nearest the file within
+  48 h. Matched releases are linked in `release_spots`. Placeholder NZBs
+  (`dummy`), gone articles (`nzb-gone`) and unreadable ones are noted on the
+  spot. Unmatched spots are matched again every hour; after
+  `SPOT_STANDALONE_AFTER_HOURS` (24) a spot becomes a release of its own
+  (source `spot`), whose NZB is the spot's (kept gzipped in `spots.nzb`).
+- **Passwords are kept apart** in `release_passwords` (release, password,
+  source: `nzb` for the NZB's `<meta type="password">`, else `spot-title` or
+  `spot-text`, and the spot), not on the catalog row; the catalog only says
+  `passworded`.
+
+## Voyager 3: window indexing (design, 2026-10-25)
+
+Branch `voyager-3` (a push to it only runs the tests; `rebuild` keeps
+running on homelab until the reset). Goal: the whole history (~1,220 B
+articles, 3% indexed on 2026-10-25) indexed and searchable on a 4 TB NVMe
+and the 15 TB HDD. The catalog costs 3.9 B of NVMe per article today
+(144 GB for 36.5 B), which puts the full history at ~4.7 TB; the target is
+≤ 2.5 B per article (~3 TB) with room for the WAL and bloat. Sources:
+LOG.md 2026-10-24 (102).
+
+What is wrong today: a release is written as soon as one range sees it,
+updated by every later range that finds more parts, merged afterwards
+(fragments, par2, names) and settled by wall-clock quiet time, which means
+little in a backfill. That churn is most of the WAL (0.6 TB a day) and the
+20 k write IOPS bursts, and leaves bloat. Half the rows are single-article
+releases, a third of them one poster's image dump (27.7 M rows, LOG.md
+(102)).
+
+1. **One cursor a group (amended 2026-10-26; was: blocks).** A group's
+   history is still fetched as ranges of 2 M articles in chunks of 100 k
+   (the spool, the corpus, the prefetcher stay) and leased as ranges, but
+   a group's ranges are indexed in article order by one worker at a time,
+   and the window below is *carried* from range to range: the next range
+   finishes what the previous one left open. The first design leased
+   blocks of 30 days of posting time, ran a group's blocks in parallel and
+   joined their edges afterwards (a block merger). The user asked why
+   (LOG.md 2026-10-25 (110)): downloading stays parallel through the
+   prefetcher and a Rust thread parses faster than a connection delivers,
+   so one worker a group loses nothing but the end of a reset, when the
+   last big group runs alone. The merger, the block cutting and the
+   `blocks` lease go. Costs: one seam a group where the history meets the
+   tail (the releases in flight then end as two incomplete halves; history
+   runs forward in one pass a group, priority only between groups), and
+   the carried state must survive a restart (5 below). Measured on six
+   consecutive ranges of a.b.ath (2016): one range is 15 hours of posting,
+   so without the carry no release would finish inside a range (LOG.md
+   2026-10-26 (111)).
+   **The group lease (2026-10-26, the user's question in LOG.md (117)):**
+   a worker thread leases a *group*, not a range, and asks the coordinator
+   for the group's next range each time it finished one; the coordinator
+   keeps one cursor a group (`group_leases`: the holder, its heartbeat,
+   the next range in article order, the snapshot it stands at). The
+   answer is the range (id, span, resume point), the rules version and the
+   stream URL — nothing about where the chunks are: the stream serves a
+   stored range's chunks from the corpus file and a live one's from the
+   spool, so a worker needs no mount and no corpus layout, and a remote
+   worker is the same binary with another URL. The overhead is a copy
+   through the node API: the corpus is ~30 B an article compressed (2016;
+   16–18 for 2021 and 2025), eight local workers at the bench's rate are
+   ~170 MB/s, which the HDD under the corpus bounds either way; a remote
+   worker is bounded by its link (100 Mbit ≈ 400k articles/s). History
+   therefore no longer waits for the corpus fill: a group's stored ranges
+   are read from disk, an unstored one is fetched live and captured into
+   the corpus on the way, and the prefetcher runs ahead of each held
+   group's cursor (plus the capture work left). A group with no planned
+   range left is the tail: the worker keeps the group and asks again
+   every minute, the planner's tail ranges fall into its order. A lease
+   whose holder went quiet for LEASE_TTL_SECONDS is taken over: the new
+   holder downloads the group's snapshot (5) and redoes the ranges since
+   it; the batch store's keys make the redo idempotent. The replay
+   harness (8) stays for offline runs; the driver on homelab goes once the
+   worker runs.
+
+2. **The window.** The worker reads a block in article order and keeps its
+   open files and releases in memory (counters, names, sizes; the segment
+   lists spooled to a file per block on the worker's disk). The *frontier*
+   is the median Date of the last 1,024 articles with a plausible date
+   (within 7 days of the frontier; 1,024 such dates in a row that agree
+   among themselves move it, the group having paused); Date headers alone
+   aren't trusted (clock skew, bogus years). 1,024, not more: the median
+   trails the newest article by half the ring, a day in a group of 500
+   articles a day. A release is **finished** when the frontier is 24 h past
+   its last part (`WINDOW_MS`): written once, with its completeness decided
+   then (every announced file there and no file short of parts; par2
+   decides later for those without a counter), never updated by the header
+   pass again. Incomplete releases are kept (user, 2026-10-24). A fill
+   posted after the window is a release of its own (the judge's `repeated`
+   rule already knows reposts). At a lease's end (the tail's poll, a worker
+   stopping; amended 2026-10-26, was: the block's end) the releases the
+   window hasn't closed on are written **provisionally** (`settled_at`
+   null, `finished: false`, flagged `edge`): a release being uploaded
+   shows in the catalog, and the carried window finishes the same row
+   later, since the release key is the identity, the group and the first
+   article; the store's finish path adds the files not sent before. No
+   merger: the carry does the joining. The settler goes: `settled_at` is
+   the finish. The **tail** needs no path of its own: its lease ends leave
+   such provisional releases, the next lease's continuation finishes them.
+   One exception (2026-10-27): a provisional release that is complete
+   (every file its counter announced found) and quiet for six hours is
+   settled by the coordinator (`Settler.settleComplete`, every five
+   minutes), as the finish would: no counted file can still come, and the
+   continuation may be hours away or never (the first run's range leases
+   left 109k provisional rows at their blocks' end edges, 606 of them
+   complete, shown as still collecting; the user, release 10697).
+
+3. **Albums.** Files of one poster in one owner group whose subjects differ
+   only by a number sequence, or share a directory-like prefix, posted
+   within the window, become one release (`source = 'album'`): its name the
+   shared template (or the prefix), its files kept by name so each stays
+   findable; at least two files, else the single stays a release. This is
+   for subjects that name no release (today's `post`/`stem` singles); a
+   poster's ten different cracks in a day stay ten releases. The stake,
+   from voyager 2's final analysis (LOG.md (112)): 77% of its 83.7 M
+   releases had one file, 50% one segment (0.3% of the segments);
+   a.b.pictures.erotica.porn alone was 27.7 M single-file releases of one
+   poster. A counter total shared by many stems of one poster within the
+   window is a batch counter, not a file count (Yenc-PP-A&A's [n/68067]):
+   `file_total` null for them, not 68,067 incomplete singles.
+   **Built 2026-10-26** (edge/src/indexer.rs, LOG.md (113)): a release the
+   window closed on with one file named by its stem or post, the name with
+   a number and no archive, volume, sidecar or random name, waits in the
+   album of its poster and template (the name's digit runs as `#`, the
+   extension apart: `#_#.jpg.data`) posted within WINDOW_MS of the album's
+   first file, at most ALBUM_MAX_FILES (1,000) a album; the album closes a
+   window after that span and is written as one release (`source: album`,
+   no file total) named by the members' shared numbers (`1481439079025_#
+   .jpg.data`), a lone member as the release it was. The batch store writes
+   the members' names as `release_texts` kind 6 (`member`, the first 20).
+   The directory-like prefix variant is not built (no case seen in the
+   corpus samples). The batch counter: a total ≥ 100 under which one poster
+   started 20+ releases within 100 batches gives none of them a file
+   total. On six a.b.ath ranges of 2016: 46 albums of 790 files among
+   2,099 chunks, most of them a subject parser miss (`[n/60] Shogun I
+   (1980) Shogun I (1980).avi.012`: the counter text taken into the name,
+   a volume each) that the album rule glues back together.
+
+4. **A slimmer catalog.** `releases` loses its text copies: `poster_id` →
+   `posters(id)` (the table gets an id; 33 B → 4), `group_set_id` →
+   `group_sets` (the distinct group combinations: 139,927 in voyager 2's
+   83.7 M releases, LOG.md (112); 85 B jsonb → 4),
+   `release_key` and `identity_key` as 20-byte `bytea` (41 B each → 20),
+   `source` a smallint. ~150 of ~430 B a row; the poster and key indexes
+   shrink with them (~12 GB of 50 today); `releases_category_newest_idx`
+   (5.3 GB, used once) goes, category browsing filters the newest index.
+   Today's 85 GB would be ~55. Cold tables move to the HDD tablespace:
+   `release_inspections` and `probe_candidates` (28 GB, written once, not
+   read by search); `probe_cache` is there already.
+
+5. **Memory and disk at the edge.** The busiest group posts ~50 M articles
+   a day; at a file per ~50 articles that is ~1 M open files of ~100 B in
+   memory per block, and their segment lists (~25 B each) in the block's
+   spool file, read back when a release is finished. Per worker thread a
+   few hundred MB at worst. The carried window (amended 2026-10-26) has to
+   survive a restart (deploys restart the workers daily): its counters and
+   names are small (kilobytes for most groups, ~100 MB for the busiest),
+   the open files' segment lists are the bulk (~1.25 GB for a day of the
+   busiest group). A snapshot at every 2 M range would be ~600 B/article
+   of writes, so the state is snapshotted at a lease's end and on a timer,
+   to a file on bulk, and a crash redoes the ranges since the last snapshot
+   from the corpus or the prefetch cache: the deterministic keys make the
+   redo idempotent. **The snapshot goes through the API** (2026-10-26):
+   `PUT /node/groups/:id/snapshot` with the serialized window (the
+   frontier, the open files and releases, the open albums, the upload
+   numbers, the batch-counter table; gzip) and the article it stands at
+   (the last batch's `next`), kept by the coordinator on bulk as
+   `<snapshot dir>/<group id>`; `GET` gives it back to the next holder. A
+   worker snapshots when it gives a group back (a stop) and every
+   SNAPSHOT_MINUTES while it holds one, always right after a batch was
+   stored, so the snapshot's article is a resume point. The replay
+   harness (8) keeps a group's window in one process without one.
+   **Built 2026-10-26** (LOG.md (117)): migration 6500 `group_leases`;
+   `app/services/group_leases.ts` (lease by the most urgent planned
+   range within LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY, the longest free
+   first; takeover after LEASE_TTL_SECONDS, the ranges from the snapshot's
+   article on planned again; next; heartbeat; release; the snapshot
+   streamed to `SNAPSHOT_DIR/<group id>.gz`); the node API's
+   `groups/lease`, `groups/:id/next|heartbeat|release|snapshot`; the
+   range stream serving a stored range's chunks from the corpus; the
+   prefetcher ahead of held groups' cursors, skipping chunks the corpus
+   has; `RangeIndexer::for_group/begin_range/snapshot/restore` (bincode,
+   the files a table, SNAPSHOT_VERSION 1); `edge-node` holding groups by
+   default (`--ranges` keeps the old path). Tests: the lease order,
+   takeover and redo, the endpoints with a corpus-served stream and the
+   snapshot through the API, the snapshot restoring a window between two
+   ranges (the two-range release one, the album joined). Deployed and
+   started 2026-10-26 22:37 (LOG.md (118)). A chunk that doesn't inflate
+   (a live reply cut short; the spool doesn't check) is reported after
+   the batch before it went out and the range asked for again from that
+   chunk: the window stays (the first run lost it, and the open batch).
+   The lease bounds alone don't select a window: the history planner's
+   oldest buckets are priority 0–2 too, so the first run also took 2026
+   history at 0 (25k ranges in 40 groups, fetched live) while 1,739 of
+   the 2016 corpus ranges sat at voyager 2's leftover priorities (done
+   when the window was marked, planned again by the reset): hence
+   `LEASE_CAPTURE_ONLY` (the lease, next and the prefetcher's run-ahead
+   take capture ranges only) and the one-off repair of the priorities.
+   The stream polls the corpus as well as the spool while it waits for a
+   chunk: the corpus fill claims chunks of the same range ahead of the
+   leased run (the prefetcher's claims are per range and chunk), stores
+   them in the corpus only, and the run leaves them there; a stream that
+   looked at the corpus once, seconds before that store, waited its whole
+   hour on the spool and the worker lost the group and its open batch
+   (three groups on the first morning). A failed stream now posts the
+   open batch before the group goes back, like a stop does, so the
+   window's snapshot survives it; only a batch the coordinator refused
+   (lease lost, too large) still loses the window.
+   **Two lanes a group (2026-10-27; migration 6600, LOG.md (119)).** A
+   group's ranges in article order through one window meant a thread
+   that leased a group for its tail range first waded through the
+   group's history (older articles come first), and a held history
+   group's tail waited behind days of its ranges; and the bounds that ran
+   the 2016 window alone (0..0, capture only) shut the tail out entirely.
+   Now `group_leases` has a row per group and *lane*: the `history` lane
+   works the ranges the bounds hand out (the windows, the history), the
+   `tail` lane the tail planner's (priority at or below the tail's,
+   whatever the bounds), each with its own holder, window, snapshot
+   (`<group>-tail.gz`) and takeover redo (the lane's ranges only, and
+   only those the window did after the snapshot: leased, done since it,
+   or read past it). The edge puts `--tail-threads` of its threads on the
+   tail lane (EDGE_TAIL_THREADS; 1 of 2 or more by default, which is
+   what homelab's 16 run with), the rest on history; the prefetcher runs ahead of every held
+   cursor. The seam a group is the same one as before, once: where the
+   history lane reaches the first range the tail lane did, it is caught
+   up. **The lease order** within a lane and priority is a mix (the
+   user, 2026-10-27): one lease starts from the group with the most
+   planned articles, the next from the longest unheld one (never held
+   first), alternating on a sequence (`group_lease_turns`), so a big
+   group snapshotted at a stop doesn't wait behind hundreds of small
+   never-held groups and a quiet group's tail is visited all the same.
+   **`coordinator:status`** (`app/services/status.ts`, `--json`) prints
+   the catalog's room against `CATALOG_MAX_NVME_GB`, the ranges by band
+   (tail, raised, each capture window, the history by thousand), the
+   last hour's indexed and stored articles, the spool, the tail backlog,
+   the held groups with their snapshots' age and size, the snapshot
+   files, and the phase-2 backlogs (provisional releases, probe
+   candidates, the pending ladder, the judge's dirty marks). **End to
+   end** (`tests/e2e`, suite `e2e`, CI builds the binary first): the Rust
+   worker against the real node API, database, spool and corpus — the
+   corpus-fill race, a failed stream and the restart that restores the
+   snapshot, a restart whose snapshot file is gone, a takeover.
+
+6. **The corpus stays as it is.** Measured on 300 unfiltered chunks
+   (30 M lines, LOG.md (102)): 43% of lines lack a "(k/N)" counter, and of
+   those 73% are random 32-hex tokens, 27% `[PRiVATE]` hashes, ~0.3% posts
+   with a readable name and no counter. The filter (crosspost, counterless)
+   loses nothing voyager 3 would index; albums work from filtered chunks
+   (the image dump's subjects carry "(1/1)"). Indexing without a counter at
+   all (obfuscated uploads, ~31% of all lines) would need bodies, and the
+   raw corpus (2,328 ranges) for a sweep.
+
+7. **Where it lives.** The window, blocks and albums are in the Rust edge
+   (`edge/`), the batch store, the planner and leases; the TypeScript
+   RangeIndexer keeps the scanner (the corpus filter, tests) and drops its
+   parity with the Rust worker for the window (the `check.ts` comparison
+   covered the range path, which goes). New migrations rebuild the schema
+   from a reset, no in-place conversion of the 81 M rows.
+
+8. **Test before the reset.** Replay the corpus (the raw ranges and the
+   captures, several B articles) with the new edge into a scratch database
+   on bulk: NVMe bytes per article (the one number that decides whether
+   4 TB is enough), the share of albums and incomplete releases, and a
+   sample of releases against today's. Speed is measured after the reset,
+   on the NVMe.
+
+9. **The reset.** Tail first, then history by priority, read from the
+   corpus where stored. Survives: `probe_cache` (HDD, by message-id),
+   `spots`, `groups` and their priorities, `nntp_shares`, the corpus.
+   Recomputed: releases, chunks, inspections, candidates, par2 data,
+   verdicts, file hashes. Resets are paused (2026-10-12); this one waits
+   for the user's go-ahead, and for the new NVMe.
+
+### Schema (built 2026-10-25; migrations 6000 and 6100)
+
+Migration 6000 *is* the reset: it truncates every table the indexing
+recomputes and refuses to run on a catalog with releases unless
+`VOYAGER3_RESET=1`, so merging `voyager-3` into `rebuild` is the go-ahead.
+Survives: `groups`, `ranges` (planned again from their start, with their
+priorities, `capture`, `est_date`, `stored_at`, `plain_chunks` and measured
+`posted_low/high`), the corpus, `probe_cache`, `spots` (unlinked from
+releases; the inspector matches them again), `nntp_shares`, `nodes`,
+`users`, the rule sets.
+
+- `releases`: created anew, 8-byte columns first: `id`, `posted_at`,
+  `last_posted_at`, `first_article`, `last_article`, `segments`, `bytes`,
+  `settled_at` (the finish; null while a tail release is provisional),
+  `inspected_at`, `first_seen_at`; `group_id`, `group_set_id`, `poster_id`,
+  `file_total`, `files`, `chunks`, `category`; `source smallint`
+  (`lib/release/source.ts`: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6
+  album); `readable`, `has_par2`, `complete`, `passworded`, `sidecar_only`,
+  `edge` (open at its block's end); `release_key` and `identity_key` as
+  20-byte `bytea`; `name`, `inner_name`. Gone: `poster`, `groups`,
+  `last_seen_at`. Indexes: pkey, release_key unique, (identity_key,
+  group_id), newest, (poster_id, newest), passworded, `(group_id,
+  identity_key) WHERE edge` for the block merger, `id WHERE settled_at IS
+  NULL` for the provisional ones. No trigram indexes: search matches
+  `release_texts`. The category trigger reads the groups from
+  `group_sets`; the judge's dirty trigger watches `poster_id` instead of
+  `poster`.
+- `pending_releases` (migrations 6300 and 6400, the HDD): the unnamed
+  releases outside the catalog (decision f): `release_key` (primary),
+  `identity_key`, `group_id`, `name` (the random stem), `poster` text, the
+  posting span, `first_article`, `last_article`, counts, `file_total`,
+  `partial`, `finished`, `tool`, `candidates` jsonb (the probe targets),
+  `inspections` jsonb (what each probe showed), `named`, `status` (0
+  waiting, 1 named and in the catalog, 2 discarded, 3 exhausted, 4 named
+  awaiting the move, 9 taken), `tries`, `created_at`, `probed_at`; partial
+  indexes on the waiting (by `created_at`) and the named (by `probed_at`).
+- `pending_signatures` (6400): per (tool, group, poster; poster '' for the
+  tool and group together) `probed`, `named`, `dropped`, `updated_at`.
+- `release_texts(release_id, kind smallint, text varchar(255))` (6200),
+  primary key on all three, one trigram index (fastupdate): every string
+  search matches against, appended by whoever learns one and never an
+  update of the releases row. Kinds (`lib/release/text_kind.ts`): 1 name
+  (the release name, stored once more so one index serves everything),
+  2 file (the prober's names inside an archive), 3 nfo (an NFO's title),
+  4 par2 (a set's name), 5 media (a container's title); at most 20 per
+  release and kind (`textsToStore`). A repeat insert is a no-op; the
+  judge's dirty mark follows an insert or delete, so a rule can react to a
+  name arriving later. Search: the trigram index yields release ids, joined
+  to `releases` by primary key (never an OR across the two tables, which
+  the planner cannot serve from indexes). `inner_name` stays on `releases`
+  as the one display string. Facets to filter on (resolution, codec,
+  language) are not texts: columns, or a `meta jsonb` with a
+  `jsonb_path_ops` index once one is proven useful.
+- `posters(id serial, poster unique)` with its trigram index;
+  `group_sets(id serial, groups jsonb unique)`, a sorted array of names.
+- `blocks` (6100): `group_id`, `start_id`, `end_id`, `next_id` (the resume
+  point), `posted_low/high`, `status`, `priority`, `node_id`, `attempts`,
+  `rule_version`, `headers`, `releases`, `leased_at`, `heartbeat_at`,
+  `done_at`, `error`; indexes todo (priority, id) and leased.
+  `ranges.block_id` points at the block; the ranges' own lease columns stay
+  until the leases move (then a migration drops them).
+- `release_chunks`, `rule_counts`, `rule_samples`: `range_id` → `block_id`.
+- Gone: `fragments`, `range_blocks`, `release_final()` (final = finished:
+  `settled_at IS NOT NULL`), the settler, the fragment merger,
+  `releases_category_newest_idx`.
+- `release_inspections` and `probe_candidates` were on `CHUNK_TABLESPACE`
+  (the HDD) with their indexes; both are back on the NVMe (migrations 4800
+  and 7300). Note LOG.md 2026-10-19 (81): the judge's fake rule read all of
+  `release_inspections` every round, 17 min on the HDD; and 2026-11-03
+  (136): the prober's catalog claim updates `probe_candidates` from random
+  points every minute, 2–3 min on the HDD beside the pending ladder's sweep.
+  What a worker reads or updates by random key every minute stays on the NVMe.
+
+### Batch store (built 2026-10-25)
+
+`app/services/batch_store.ts`: every release chunk of a batch is one
+release, inserted once. Its keys (`lib/release/keys.ts`): `identity_key`
+the identity's SHA-1 as 20 bytes; `release_key` the SHA-1 of the identity,
+the owner group and the release's first article, so it is deterministic
+(no numbering under a group lock, no catalog lookups before writing) and a
+repost, starting at another article, is a release of its own. Its poster
+and group set by id (`app/services/catalog_refs.ts`: insert what's new,
+read the ids, cache them per process; the tests forget the cache per
+test), its source as a code, its name into `release_texts` (kind 1). The
+chunk's own counts go in as the release's; `complete` is decided then from
+the counter. A finished release's probe candidates are claimable at once
+(`ready`, `unreadable`); a provisional tail release (`finished: false` on
+the chunk, `settled_at` null) is finished by a later chunk of the same
+release key, which adds the files not sent before (sums, one more chunk),
+sets `settled_at` and makes its candidates claimable; a finished release
+sent again is left as it is (`ON CONFLICT … WHERE settled_at IS NULL`).
+Gone: the join of a chunk into an earlier release of its key (the window
+keeps an upload together; the block merger joins what a block's end cut),
+the join across groups (FTD's spread uploads: a gap, to be handled by the
+block merger or the par2 merger across groups), the numbering of release
+keys, the group lock, `fragments` (ignored when sent; the Rust edge still
+sends them). The chunk type carries `finished` and `edge` (both optional;
+the Rust edge doesn't send them yet, so its releases are finished, not
+edges). Interim until blocks are leased: chunks, rule counts and samples
+are keyed by `ranges.block_id`, and a range without a block gets a block
+of its own in the store (`blockOf`).
+
+Readers moved with it: search joins `posters` and `group_sets`, matches
+words in `release_texts` (`r.id IN (SELECT release_id … ILIKE)`), searches
+posters by id, filters `source` by code and returns the source name; the
+par2 merger, the sidecar folder and the judge's scraped rule use
+`poster_id` (`release_merges.from_key` holds the key as hex); the spot
+inspector matches by bytea identity keys and inserts its spot releases by
+id references with their title in `release_texts`; the spotter and the NZB
+service compare the source code. Tests insert fixtures through
+`tests/functional/helpers/releases.ts` (`insertRelease`: text keys, poster
+and groups by name, source by name). The unit and functional suites pass.
+
+### The window in the Rust edge (built 2026-10-25)
+
+`edge/src/indexer.rs`: the scanner and the file aggregator as before; then
+the window instead of the range path. `Frontier` keeps a ring of the last
+`FRONTIER_ARTICLES` (1,024) plausible dates (within `PLAUSIBLE_MS`, 7
+days, of the frontier; 1,024 implausible dates in a row that agree move
+it), recomputed once per chunk; established after 64 dated articles
+(`settle` at the block's end for smaller blocks); its first value is the
+block's start in posting time. Per chunk: completed files, then the files
+whose last part is before frontier − `WINDOW_MS` (24 h; undated ones idle
+for `idle` chunks), go into the open release of their key whose posting
+span is within `UPLOAD_GAP_MS` (3 h) of theirs, else start one (the
+upload number per key, distinct within a batch); releases whose last part
+is before frontier − `WINDOW_MS` are judged and written `finished`, as an
+`edge` when posted within `WINDOW_MS` of the block's start. At the end
+(`finish`) everything left is written: finished when the window closed on
+it, else `finished: false, edge: true`. A file short of parts stays in
+its release and sets `partial` (the store: `complete = false`). A lone
+article of a multi-part random file is still counted `per-article`, not
+written; before the block's end a scrambled name counts as random too
+(`looks_scrambled`: a random subject and poster per article, 160k of
+alt.binaries.storage's releases; LOG.md (122)).
+
+**Single-archive rules (built 2026-10-31; LOG.md (123)-(127)).** The judge
+(`names::judge`, `RuleSet.judge`) gets each file's articles seen and their
+bytes, and whether the release is *settled* (the window closed on it: a
+par2 can still join an open one; the start edge doesn't count, since a par2
+follows its rar and 7% of the catalog's releases are start edges). A settled
+release of one unsplit zip, rar or 7z and nothing else is dropped as a
+**stub archive** (`stub-archive`, key: the legs tripped) when posted to
+2+ groups, under 100 MB (the fakes stop at ~90 MB; hash-named uploads,
+real albums and passworded videos above it don't), and (A) its articles
+are ≤ 256 KiB with the release ≥ 1 MB (Power-Post's 250 kB default; a
+one-article release always has small articles, which exempts e-books and
+sidecars), or (X) posted to 4+ groups, or (W) by a webmail poster. A
+ROT13 volume name (".cnegN.rar" is ".partN.rar": a.b.erotica's passworded
+videos, LOG.md (128)) is a volume, not a single archive. A release
+touching a music, e-book, comics, tabs or fonts group is exempt: those
+communities post real single small archives the same way (karaoke zips to
+a.b.sounds.karaoke + a.b.test with Power-Post's articles; 16 of 32,308
+known fakes lost, LOG.md (127)). These are the
+exe-stub and recycled-installer floods under film and software names:
+55% of the catalog on 2026-10-30, 99.3% of the known fakes, 0 known
+content in the 217k sample, verified on 1,500 decoded articles. A settled
+single archive of one article and 100 kB–1 MB under a release-like,
+unscrambled name without spaces and without an e-book format in it is
+dropped as a **nonsense piece** (`nonsense-piece`, key: the upload tool):
+the newznzb.* pieces (scene-styled or random with a "-XX" suffix), 9% of
+the catalog; under 100 kB are cue/log sidecars and adverts, and the names
+with spaces are books and albums. Below that, a settled single archive of
+one article and 2–100 kB under a release-like name (spaces allowed), outside
+the exempt groups and without an e-book, subtitle, crack/keygen/trainer,
+cover or sidecar word, is dropped as a **tiny archive** (`tiny-archive`,
+key: the upload tool; LOG.md (131), (133)): the 2015–17 flood's RARs
+holding "<title>.exe" at 2–60 kB, 1.66M catalog rows from a dozen posters
+(12 of 12 fetched fake); the word exemptions keep the real tiny archives
+(subtitles, cracks, trainers) at the price of ~50k fakes named "Incl
+Keygen" that the prober judges later. The rules are switched by the
+rule set (`dropStubArchives`, `dropNonsensePieces`, `dropTinyArchives`),
+sampled like every drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer
+judges as each batch completes files, so it passes settled without the
+window's knowledge). Gone: fragments and the cut-off, `uploads_of` (online now),
+`check.ts` and the fixture comparison (the parity with the TypeScript
+indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count
+(incomplete releases are kept). The wire carries `finished`, `edge` and
+`partial`; the store reads them (`partial` into `complete`). Memory: the
+window holds a day of a group's segments in memory (the busiest group's
+~50 M); the spool file per block of the design is not built yet.
+
+### The replay harness (built 2026-10-26)
+
+`node ace replay:corpus --group <name> [--dir CORPUS_DIR] [--from article]
+[--to article] [--limit ranges] [--batch-chunks 5] [--no-finish]` (commands/
+replay_corpus.ts; design 8): a group's complete corpus ranges in article
+order through one window in the Rust edge (`edge-bench --group <dir>
+--rules rules.json --wire -`: the window carried from range to range, the
+end finished after the last range; `edge/src/lease.rs` has the lease JSON
+the worker and the bench share) and the batch store into a scratch
+catalog, as the worker would store them: each range leased to a node named
+`replay`, its batches stored at its resume point (pulled from the bench's
+stdout as the store keeps up), the range finished when its last batch is
+in. It refuses a database not named `*_replay` or `*_scratch`.
+The rules are the scratch catalog's current set and group priorities:
+copy `groups` and `rule_sets` from the catalog for a faithful replay. The
+report: releases finished, provisional, edges; complete, incomplete,
+undecided; readable, with par2; files, segments as a share of the
+headers; bytes per header of the NVMe tables (releases, release_texts,
+posters, group_sets, rule counts and samples) and of those on
+CHUNK_TABLESPACE (release_chunks, probe_candidates, release_inspections).
+The app image carries `edge-bench` for it (Dockerfile); on homelab the
+scratch catalog is a second database (`usenet_replay`, default tablespace
+`nvme` like the catalog, the chunk tables on `pg_default`; migrated with
+the voyager-3 image built from the deploy cache's repo, `groups` and
+`rule_sets` copied over), the corpus mounted read-only.
+`deploy/replay-window.sh` runs a window there: each group whose window
+ranges are all stored is replayed once with the window's bounds as
+`--from`/`--to` (the corpus holds other windows of the same group), 4
+groups at a time, logs and `.done` markers in `.cache/replay/`, looping
+until the window is done (LOG.md 2026-10-26 (111) has the first local
+numbers, (112) the homelab run). **Deployed 2026-10-26** (LOG.md (116)):
+voyager 3 runs on homelab without its edge worker (`.cache/no-edge`:
+deploy.sh and the nightly resume leave it stopped, since range by range
+it would split releases at every edge with no merger left), and history
+comes from the corpus through this driver into the catalog itself
+(`DB=usenet_rebuild OUT=.cache/replay-catalog`, the harness allowed by
+`REPLAY_INTO_CATALOG=1`), the prober's ladder, the merger and the judge
+working on it as it fills; the scratch run's numbers (63 groups) were
+read before (LOG.md (115)). The tail waits for the window worker.
+
+Next: the replay on homelab over the 2016 window (the NVMe bytes per
+header for real), the window worker (design 1 and 5: group leases, the
+stream serving stored chunks, the snapshot through the API; being built,
+LOG.md (117)), the prober writing `release_texts` kind 2, the spot/NZB
+match on pending keys (decision f), the tail's provisional writes at a
+range end (the window's open releases shown while they upload: the files
+sent provisionally marked, the finish sending the rest). Gone from the list: block
+leases, the planner and the block merger (design 1, amended).
 
 ## Decisions
 
@@ -192,3 +892,99 @@ c. Group list: editable, owner by priority number (above). **Accepted.**
 d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
 e. Headers through the coordinator's relay, with prefetch (part 1).
    **Accepted.**
+f. **Nothing enters the catalog without a usable name** (amended
+   2026-10-26, the user's direction; before: random-named releases keep
+   no chunk but their rows). A release the window closed on whose files
+   all have random names (`isUnreadableName`, `names::is_unreadable_name`:
+   a name read token by token, so separators, spaces and random
+   "extensions" don't hide a scramble; since 2026-11-02, LOG.md (129),
+   when 15% of the catalog's "readable" names were "Zx5AjNaK-mh7CFg6Hf8uu"
+   and "Description - Lt5nHjlo3RiuBb3C") is no release yet: it goes to `pending_releases`
+   (migration 6300, the HDD) with its keys, counts, article span, upload
+   tool and probe targets, no row in `releases`, `release_texts` or
+   `posters`, no chunk. The pending ladder works them **from least to most
+   effort**: (0) no fetch: a signature verdict (tool and group, poster)
+   learned from earlier probes, a spot or NZB that names the message-ids;
+   (1) a par2 main file's first article (it names the files); (2) the first
+   article of the first file (the yEnc name, an archive's listing in its
+   first 16 KiB); (3) a 7z's tail; then nothing more. A probe that yields
+   a usable name moves the release into the catalog, its segments read
+   again from the corpus (the range stored) or the server; one that
+   yields none counts against its signature, and a signature with
+   ≥100 probes and ≤0.5% named (voyager 2's garbage learner, LOG.md
+   2026-10-08 (5)) is **discarded** from then on, a sample of it still
+   probed to notice a change. **By class** (2026-11-02, LOG.md (130)): a
+   pending release's class is its tool, group, size bucket, name shape and
+   partial flag (`lib/release/classes.ts`, `names::pending_class`, the SQL
+   `pending_class()`; `pending_releases.klass`), and `pending_classes` keeps
+   each class's climbs and names; the claim takes the classes with the best
+   record first (`classScore`, a prior for the unexplored, at most a quarter
+   of a claim from one class), the cheapest target article first within one
+   (`target_bytes`), and the oldest of the rest; a class with the learner's
+   thresholds is dropped like a signature, and the rule set's
+   `droppedClasses` keep its releases out of the pending store at the edge,
+   the explored sample excepted. A probed file whose first 16 KiB and size
+   match a named catalog release's (`file_hashes`) takes that name without
+   another fetch (`app/services/hash_twins.ts`), on the ladder and for
+   catalog releases without one. The reason, in the 2016 replay: one poster's
+   hash-named pieces ("Bob", 11 aliases, 3.2 M releases in five small
+   groups) were 76% of the rows and three quarters of the NVMe bytes for
+   11% of the segments and nothing anyone can find; the full history has
+   725 M random-named files (LOG.md 2026-10-09). Built 2026-10-26: the
+   edge sends an unnamed release with an empty payload and its tool, the
+   batch store writes `pending_releases` (LOG.md (114)); a finished release
+   the edge found no target for (the edge's `candidates_of`: par2, archive
+   start, 7z tail, else the first random-named file as the lone target;
+   LOG.md (140)) is written exhausted at once; the ladder in the
+   prober (`app/services/pending_ladder.ts`, LOG.md (115)): a claim takes
+   the oldest waiting releases' next target (par2, archive, lone, 7z-tail
+   in that order), at least PENDING_SHARE (25%) of each claim next to the
+   catalog's own candidates, answered from `probe_cache` when the article
+   was read before; `usableName` (lib/probe/inspect.ts) takes the largest
+   named file of a par2 set or a listing, or a lone file's yEnc name, that
+   is neither packaging nor random; a name sets status 4, no name the next
+   target or status 3 (exhausted); each finished climb counts on
+   `pending_signatures` for (tool, group, poster) and (tool, group), and a
+   signature with PENDING_MIN_PROBES (100) climbs and at most
+   PENDING_MAX_NAMED (0.5%) named is dropped: a sweep beside the lanes
+   (every five minutes, in its own loop) discards the waiting releases
+   of each dropped signature and class once (`swept_at`; the edge drops
+   their new releases itself), the rules' `samplePercent` of them left to
+   climb, each by an index (`pending_releases_signature_idx` for a
+   poster's rows: tested on the rows, a signature was a scan of the whole
+   table; LOG.md (133), (134)). The batch store, the record, the claim
+   and the sweep lock pending rows in one order, by key and the counters
+   after (`lockOrder` for the multi-row statements; the record and the
+   sweep lock their rows with `SELECT … ORDER BY release_key FOR UPDATE`
+   first, since an UPDATE's join or `= ANY(subquery)` locks in plan
+   order), and a flush that deadlocks anyway keeps its outcomes for the
+   next (LOG.md (134), (137)). The namer
+   (`app/services/pending_namer.ts`, in the prober's process): a named
+   release's article span read again from the corpus (the stored range's
+   chunks) or by XOVER on a connection of its own, a chunk of articles
+   at a time (one string of a wide span topped V8's limit; spans over 3M
+   articles are given up), through the TypeScript
+   indexer with `partialFiles` (a file given up short of parts stays a
+   partial file of its release, as the Rust window writes it; without it
+   the range path's fragment rule left nothing to find: by 2026-10-29
+   about 52k of 210k named releases failed, all partial), written as a release under the pending key with `source`
+   weak-stem, the name as `inner_name`, its chunk and its candidates (the
+   prober answers those from `probe_cache`: inspections, par2 sets, hashes,
+   the password flag). Not built: the spot/NZB match on pending keys (step
+   0 of the ladder beyond signatures), and a signature by poster alone
+   across groups.
+g. The account's NNTP connections shared by priority (`nntp_shares`): the
+   coordinator (headers) before the prober, each between its minimum and
+   maximum, 5 kept for the spotter, the spot inspector, the plain
+   connection and reconnects. A process grows only into connections the
+   others closed, so the account never goes over its limit. Replaces the
+   fixed split (38 + 8), under which the prober stayed at 8 while history
+   was paused and the coordinator's connections idle (LOG.md 2026-10-18
+   (75)). **Accepted** 2026-10-18.
+h. Idle NNTP connections closed: one with nothing to do for
+   `NNTP_IDLE_CLOSE_SECONDS` (60) is closed and opened again with work
+   (the coordinator's pool on its next sync, the plain one too; a dry
+   prober lane; the spotter between rounds; the spot inspector with no
+   spots). Eweka dropped the quiet ones and the next command failed
+   ("Spot inspection failed: NNTP connection closed"), and idle ones took
+   the account's connections from downloads. **Accepted** 2026-10-22.

adonisrc.ts

@@ -53,6 +53,7 @@ export default defineConfig({
     () => import('@adonisjs/cors/cors_provider'),
     () => import('@adonisjs/auth/auth_provider'),
     () => import('#providers/api_provider'),
+    () => import('@adonisjs/static/static_provider')
   ],
 
   /*
@@ -90,6 +91,12 @@ export default defineConfig({
         name: 'functional',
         timeout: 30000,
       },
+      // The Rust edge worker against the real node API (tests/e2e): needs edge/target/release/edge-node.
+      {
+        files: ['tests/e2e/**/*.spec.{ts,js}'],
+        name: 'e2e',
+        timeout: 120000,
+      },
     ],
     forceExit: false,
   },
@@ -103,7 +110,10 @@ export default defineConfig({
   | the production build.
   |
   */
-  metaFiles: [],
+  metaFiles: [{
+    pattern: 'public/**',
+    reloadServer: false,
+  }],
 
   hooks: {
     init: [

app/controllers/newznab_controller.ts

@@ -0,0 +1,116 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import env from '#start/env'
+import NzbService from '#services/nzb_service'
+import SearchService from '#services/search_service'
+import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
+
+const LIMITS = { max: 100, default: 50 }
+
+/** Search functions and the category each defaults to when `cat` is absent. */
+const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
+  'search': undefined,
+  'tvsearch': 5000,
+  'tv-search': 5000,
+  'movie': 2000,
+  'movie-search': 2000,
+  'music': 3000,
+  'audio': 3000,
+  'book': 7000,
+}
+
+/** The configured key, if any. */
+function configuredApiKey(): string | undefined {
+  const value: any = env.get('NEWZNAB_API_KEY')
+  const key = typeof value === 'string' ? value : value?.release()
+  return key || undefined
+}
+
+function int(value: unknown): number | undefined {
+  return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined
+}
+
+/**
+ * Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1):
+ * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
+ * Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
+ */
+export default class NewznabController {
+  async handle({ request, response }: HttpContext) {
+    const qs = request.qs()
+    const t = String(qs.t ?? '')
+    const server = this.server(request)
+    const xml = (body: string, type = 'application/xml') => {
+      response.header('Content-Type', `${type}; charset=utf-8`)
+      return response.send(body)
+    }
+
+    const apiKey = configuredApiKey()
+    if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
+    if (t === 'caps') return xml(capsXml(server, LIMITS))
+
+    if (t === 'get') {
+      const id = int(qs.id)
+      if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
+      const nzb = await NzbService.build(id)
+      if (!nzb) return xml(errorXml(300, 'No such item'))
+      const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
+      response.header('Content-Disposition', `attachment; filename="${filename}"`)
+      return xml(nzb.xml, 'application/x-nzb')
+    }
+
+    if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
+
+    const requested = String(qs.cat ?? '')
+      .split(',')
+      .map((c) => int(c.trim()))
+      .filter((c): c is number => c !== undefined)
+    const fallback = SEARCH_FUNCTIONS[t]
+    const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : [])
+    const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
+    const offset = int(qs.offset) ?? 0
+
+    // ID-only lookups (tvdbid, imdbid, …) aren't supported: an empty result rather than everything.
+    const q = typeof qs.q === 'string' ? qs.q.trim() : ''
+    const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
+    const result = idOnly
+      ? null
+      : await SearchService.search({
+          q,
+          categories,
+          readable: true,
+          notIncomplete: true,
+          maxAgeDays: int(qs.maxage),
+          season: int(qs.season),
+          episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
+          sort: 'newest',
+          perPage: limit,
+          offset,
+        })
+
+    const items = (result?.releases ?? []).map((r) => {
+      const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
+      return {
+        id: r.id,
+        // An obfuscated release by what the prober found inside, without its extension (what clients parse).
+        title: r.readable || !r.innerName ? r.name : r.innerName.replace(/\.[a-z0-9]{2,4}$/i, ''),
+        category: r.category ?? 8010,
+        size: r.size,
+        files: r.fileCount,
+        postedAt: r.postedAt ? new Date(r.postedAt) : null,
+        poster: r.poster,
+        groups: r.groups ?? [],
+        // From the prober (phase 2): encrypted archive entries or headers.
+        password: r.passworded === true,
+        season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
+        episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
+      }
+    })
+    return xml(resultsXml(server, items, offset, result?.total ?? 0), 'application/rss+xml')
+  }
+
+  private server(request: HttpContext['request']): NewznabServer {
+    const proto = request.header('x-forwarded-proto') ?? 'http'
+    const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
+    return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey: configuredApiKey() }
+  }
+}

app/controllers/node_controller.ts

@@ -1,13 +1,15 @@
 import type { HttpContext } from '@adonisjs/core/http'
 import { Readable } from 'node:stream'
 import { setTimeout as sleep } from 'node:timers/promises'
+import db from '@adonisjs/lucid/services/db'
 import Leases, { LeaseLost } from '#services/leases'
+import GroupLeases, { LANES } from '#services/group_leases'
 import BatchStore, { StaleBatch } from '#services/batch_store'
-import Spool from '#services/spool'
-import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
+import Spool, { type ChunkSpan } from '#services/spool'
+import Corpus from '#services/corpus'
+import coordinatorConfig from '#config/coordinator'
+import { KEEPALIVE, encodeFrame, fromWire, type Lane, type WireBatch } from '#lib/edge/index'
 
-/** How long the stream waits for a chunk the prefetcher hasn't spooled yet. */
-const CHUNK_WAIT_MS = 15 * 60 * 1000
 /** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
 const KEEPALIVE_MS = 20_000
 
@@ -26,18 +28,109 @@ export default class NodeController {
     return lease
   }
 
-  /** The range's chunks from its resume point, waiting for those still being fetched. */
+  // Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
+  // ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
+  // `?lane=tail` is the group's tail cursor (its own holder, window and snapshot); history by default.
+
+  /** The lane the request names (`?lane=`), or null after a 400 for one that isn't a lane. */
+  private lane(ctx: HttpContext): Lane | null {
+    const lane = ctx.request.qs().lane ?? 'history'
+    if (LANES.includes(lane)) return lane
+    ctx.response.badRequest({ error: `lane: ${LANES.join(' or ')}, not "${lane}"` })
+    return null
+  }
+
+  async leaseGroup(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
+    const lease = await GroupLeases.lease(this.node(ctx), lane)
+    if (!lease) return ctx.response.noContent()
+    return lease
+  }
+
+  async nextRange(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
+    try {
+      const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx), lane)
+      if (!next) return ctx.response.noContent()
+      return next
+    } catch (error) {
+      if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
+  async groupHeartbeat(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId, lane))
+  }
+
+  async releaseGroup(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId, lane))
+  }
+
+  /**
+   * PUT /groups/:id/snapshot?next=N: the window's serialized state standing at article N, streamed to disk as sent
+   * (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
+   */
+  async putSnapshot(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
+    const next = Number(ctx.request.qs().next)
+    if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
+    const raw = ctx.request.raw()
+    const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body, lane))
+  }
+
+  async getSnapshot(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
+    try {
+      const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx), lane)
+      if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
+      ctx.response.header('Content-Type', 'application/octet-stream')
+      ctx.response.header('Content-Length', String(snapshot.size))
+      ctx.response.header('X-Snapshot-Next', String(snapshot.next))
+      return ctx.response.stream(snapshot.stream)
+    } catch (error) {
+      if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
+  /**
+   * The range's chunks from its resume point: a stored range's from the corpus (a worker needs no mount; REBUILD.md
+   * design 1), else from the spool, waiting for those still being fetched.
+   */
   async stream(ctx: HttpContext) {
     const nodeId = this.node(ctx)
     const rangeId = Number(ctx.params.id)
     const range = await this.leased(ctx, rangeId, nodeId)
     if (!range) return
     const spans = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }, Number(range.next_id))
+    const group = range.capture ? await db.from('groups').where('id', range.group_id).select('name').first() : null
+    const corpusRange = group ? { group: group.name as string, start: Number(range.start_id), end: Number(range.end_id) } : null
+    // The chunk from the corpus or the spool, whichever has it. Both are polled while waiting: the corpus fill
+    // claims a chunk ahead of the leased run (the prefetcher's claims are per range and chunk), stores it in the
+    // corpus only, and the run then leaves it there — a stream that looked at the corpus once, before that
+    // store, waited the whole hour on the spool (three groups on 2026-10-27, LOG.md (118)).
+    const stored = async (span: ChunkSpan) =>
+      corpusRange && (await Corpus.has(corpusRange, span)) ? Corpus.read(corpusRange, span) : null
     async function* frames() {
       for (const span of spans) {
-        const waitUntil = Date.now() + CHUNK_WAIT_MS
+        const waitUntil = Date.now() + coordinatorConfig.streamChunkWaitMs
         let quietSince = Date.now()
-        while (!(await Spool.has(rangeId, span))) {
+        let data = await stored(span)
+        while (!data) {
+          if (await Spool.has(rangeId, span)) {
+            data = await Spool.read(rangeId, span)
+            break
+          }
           if (Date.now() > waitUntil) throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} wasn't spooled in time`)
           if (Date.now() - quietSince >= KEEPALIVE_MS) {
             quietSince = Date.now()
@@ -45,8 +138,9 @@ export default class NodeController {
             await Leases.heartbeat(rangeId, nodeId)
           }
           await sleep(500)
+          data = await stored(span)
         }
-        yield encodeFrame({ ...span, data: await Spool.read(rangeId, span) })
+        yield encodeFrame({ ...span, data })
         // Streaming keeps the lease alive.
         await Leases.heartbeat(rangeId, nodeId)
       }
@@ -71,7 +165,11 @@ export default class NodeController {
   }
 
   async finish(ctx: HttpContext) {
-    return this.lifecycle(ctx, (rangeId, nodeId) => Leases.finish(rangeId, nodeId))
+    return this.lifecycle(ctx, async (rangeId, nodeId) => {
+      const range = await Leases.leased(rangeId, nodeId)
+      await Leases.finish(rangeId, nodeId)
+      await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers), GroupLeases.laneOf(Number(range.priority)))
+    })
   }
 
   /** POST /ranges/:id/corrupt { from }: a chunk that doesn't inflate; the range goes back. */

app/controllers/releases_controller.ts

@@ -1,27 +1,61 @@
 import type { HttpContext } from '@adonisjs/core/http'
-import db from '@adonisjs/lucid/services/db'
 import NzbService from '#services/nzb_service'
+import SearchService, { type SortOrder } from '#services/search_service'
+import { expandCategories } from '#lib/newznab/index'
 
-/** The catalog (REBUILD.md, part 9): a name search and a release's NZB. */
+const flag = (value: unknown) => value === '1' || value === 'true' || value === true
+
+/** A size in MB from the query string, as bytes. */
+function megabytes(value: unknown): number | undefined {
+  const mb = Number(value)
+  return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined
+}
+
+/** The catalog for the web interface (REBUILD.md, part 9). */
 export default class ReleasesController {
-  /** Releases whose name has every word of `q` (dots, dashes, underscores and spaces alike), newest first. */
+  /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
   async index({ request }: HttpContext) {
-    const q = String(request.input('q', '')).trim()
-    const limit = Math.min(Number(request.input('limit', 50)) || 50, 200)
-    const query = db
-      .from('releases')
-      .select('id', 'name', 'poster', 'groups', 'posted_at', 'files', 'file_total', 'bytes', 'complete', 'readable')
-      .orderBy('posted_at', 'desc')
-      .limit(limit)
-    for (const word of q.split(/[\s._-]+/).filter(Boolean)) query.whereILike('name', `%${word.replace(/[\\%_]/g, '\\$&')}%`)
-    return { releases: await query }
+    const qs = request.qs()
+    const categories = String(qs.cat ?? '')
+      .split(',')
+      .map((c) => Number(c.trim()))
+      .filter((c) => Number.isInteger(c) && c > 0)
+    return SearchService.search({
+      q: typeof qs.q === 'string' ? qs.q : undefined,
+      categories: categories.length ? expandCategories(categories) : undefined,
+      readable: flag(qs.readable),
+      complete: flag(qs.complete),
+      unlocked: flag(qs.unlocked),
+      source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
+      poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
+      minSize: megabytes(qs.min_size),
+      maxSize: megabytes(qs.max_size),
+      sort: qs.sort as SortOrder | undefined,
+      page: Number(qs.page) || 1,
+      perPage: Number(qs.per_page) || 50,
+      // The page shows "1,000+"; counting further made filter clicks slow.
+      countLimit: 1000,
+    })
   }
 
+  /** GET /api/v1/releases/:id, with its files */
+  async show({ params, response }: HttpContext) {
+    const release = await SearchService.release(Number(params.id))
+    return release ?? response.notFound({ error: 'Release not found' })
+  }
+
+  /** GET /api/v1/releases/:id/nzb */
   async nzb({ params, response }: HttpContext) {
     const nzb = await NzbService.build(Number(params.id))
-    if (!nzb) return response.notFound({ error: 'No such release' })
+    if (!nzb) return response.notFound({ error: 'Release not found' })
+    const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
     response.header('Content-Type', 'application/x-nzb')
-    response.header('Content-Disposition', `attachment; filename="${nzb.name.replace(/["\\/]/g, '_')}.nzb"`)
-    return nzb.xml
+    response.header('Content-Disposition', `attachment; filename="${filename}"`)
+    return response.send(nzb.xml)
+  }
+
+  /** GET /api/v1/stats */
+  async stats() {
+    return SearchService.stats()
   }
 }

app/services/batch_store.ts

@@ -1,64 +1,118 @@
 import db from '@adonisjs/lucid/services/db'
+import { pruneSevenZipTails } from '#services/seven_zip_tails'
 import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
 import type { EdgeBatch, ProbeCandidate, ReleaseChunk } from '#lib/edge/index'
 import ChunkPartitions from '#services/chunk_partitions'
+import CatalogRefs, { groupSetKey, posterText } from '#services/catalog_refs'
+import { decodeChunk } from '#lib/chunks/index'
+import { SIDECAR } from '#services/sidecar_folder'
+import { identityKey, releaseKey } from '#lib/release/keys'
+import { sourceCode } from '#lib/release/source'
+import { uploadTool } from '#lib/rules/index'
+import { PENDING, byEffort, lockOrder } from '#services/pending_ladder'
+import { TEXT_KINDS, TEXT_MAX, textsToStore } from '#lib/release/text_kind'
+
+/**
+ * A small new release (at most 3 files, under 1 MB) whose files are all sidecars (.nfo, .sfv, .nzb, …): the sidecar
+ * fold's candidates (releases.sidecar_only). Its file list decoded here, small as it is.
+ */
+function sidecarOnly(r: ReleaseChunk): boolean | null {
+  if (r.files > 3 || r.bytes >= 1_000_000) return null
+  const files = decodeChunk(r.payload)
+  return files.length > 0 && files.every((f) => SIDECAR.test(f.name))
+}
 
 export class StaleBatch extends Error {}
 
 export interface StoredBatch {
+  /** Release chunks in the batch. */
   releases: number
+  /** Releases the batch started. */
   newReleases: number
+  /** Provisional (tail) releases the batch finished. */
+  finalized: number
   candidates: number
-  fragments: number
+  /** Unnamed releases written to the pending store instead of the catalog (decision f). */
+  pending: number
 }
 
+/** A chunk's slot in its batch: the key and the upload the edge numbered it with (ProbeCandidate points at it). */
+const slot = (key: string, upload: number) => `${key}\n${upload}`
+
 /**
- * Stores one batch of a leased range (REBUILD.md, parts 3–7), in one
- * transaction: release aggregates in the catalog, one chunk per release,
- * probe candidates, rule counts and samples, edge fragments, and the
- * range's resume point. A batch that doesn't start at the resume point was
+ * Stores one batch of a leased range (REBUILD.md, "Voyager 3: window indexing", 2 and 4) in one transaction: each
+ * release chunk is a whole release the edge's window finished, written once (its poster and group set by id, its
+ * keys as bytea, its name into release_texts), with one chunk of its files, its probe candidates, and the batch's
+ * rule counts and samples; then the range's resume point moves. A batch that doesn't start at the resume point was
  * sent twice or too late, and is refused (StaleBatch), so the sums stay right.
+ *
+ * A tail release comes twice: provisionally (finished: false, settled_at null, its candidates waiting) and then
+ * finished, under the same release key (the identity, the group and its first article: lib/release/keys.ts), with
+ * the files not sent before (a chunk's counts are its own; the finish adds them up); the finish is the one update a
+ * release gets from the header pass. A finished release sent again is left as it is.
+ *
+ * Interim, until blocks are leased (the next step): chunks, counts and samples are keyed by the range's block
+ * (ranges.block_id), and a range without one gets a block of its own here.
  */
 class BatchStore {
   async store(rangeId: number, nodeId: number, batch: EdgeBatch): Promise<StoredBatch> {
     await ChunkPartitions.ensure()
     return db.transaction(async (trx) => {
-      const range = await trx.from('ranges').where('id', rangeId).forUpdate().first()
+      // One batch of a range at a time, without holding the range's row for the whole transaction: the
+      // prefetcher's and heartbeats' updates of it waited on it (0.5 s on average, NNTP connections idle;
+      // LOG.md 2026-10-09 (13)). The resume point is checked again when it moves, at the end.
+      await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [
+        rangeId,
+      ])
+      const range = await trx.from('ranges').where('id', rangeId).first()
       if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
         throw new StaleBatch(`Range ${rangeId} is not leased to this node`)
       if (Number(range.next_id) !== batch.from)
         throw new StaleBatch(
           `Range ${rangeId} is at ${range.next_id}, the batch starts at ${batch.from}`
         )
+      const blockId = await this.blockOf(trx, range)
       const result: StoredBatch = {
         releases: batch.releases.length,
         newReleases: 0,
+        finalized: 0,
         candidates: 0,
-        fragments: batch.fragments.length,
+        pending: 0,
       }
 
+      // Nothing enters the catalog without a usable name (decision f): an unnamed release (no readable file name)
+      // goes to the pending store with its probe targets, the named ones to the catalog.
+      const named = batch.releases.filter((r) => r.readable)
+      const unnamed = batch.releases.filter((r) => !r.readable)
       const written = await this.writeReleases(
         trx,
-        rangeId,
+        { groupId: Number(range.group_id), blockId },
         batch.from,
-        batch.releases,
+        named,
         batch.candidates
       )
       result.newReleases = written.newReleases
+      result.finalized = written.finalized
       result.candidates = written.candidates
+      result.pending = await this.writePending(
+        trx,
+        Number(range.group_id),
+        unnamed,
+        batch.candidates
+      )
 
       if (batch.counts.length > 0) {
         await trx.rawQuery(
-          `INSERT INTO rule_counts (range_id, rule, key, articles, files)
+          `INSERT INTO rule_counts (block_id, rule, key, articles, files)
            SELECT ?, v.rule, v.key, v.articles, v.files FROM jsonb_to_recordset(?::jsonb) AS v(rule text, key text, articles bigint, files bigint)
-           ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
-          [rangeId, JSON.stringify(batch.counts)]
+           ON CONFLICT (block_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
+          [blockId, JSON.stringify(batch.counts)]
         )
       }
 
       for (const s of batch.samples) {
         await trx.insertQuery().table('rule_samples').insert({
-          range_id: rangeId,
+          block_id: blockId,
           rule: s.rule,
           rule_key: s.ruleKey,
           release_key: s.key,
@@ -69,114 +123,304 @@ class BatchStore {
         })
       }
 
-      for (let i = 0; i < batch.fragments.length; i += 2000) {
-        const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
-          range_id: rangeId,
-          file_key: f.fileKey,
-          article_number: Math.min(...f.segments.map((s) => s[3])),
-          segments: f.segments.length,
-          fragment: JSON.stringify(f),
-        }))
-        await trx.insertQuery().table('fragments').multiInsert(rows)
-      }
-
-      await trx
+      const moved = await trx
         .from('ranges')
         .where('id', rangeId)
+        .where('status', 'leased')
+        .where('node_id', nodeId)
+        .where('next_id', batch.from)
         .update({
           next_id: batch.next,
           headers: db.raw('headers + ?', [batch.headers]),
+          // The range's posting time span: the batches' trimmed spans, min and max (least and greatest skip nulls).
+          posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [
+            batch.postedLow ?? null,
+          ]),
+          posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [
+            batch.postedHigh ?? null,
+          ]),
           heartbeat_at: new Date(),
         })
+      if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1))
+        throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
       return result
     })
   }
 
+  /** The range's block; a range without one (no planner yet) is a block of its own, leased as the range is. */
+  async blockOf(trx: TransactionClientContract, range: any): Promise<number> {
+    if (range.block_id) return Number(range.block_id)
+    const [block] = await trx
+      .table('blocks')
+      .insert({
+        group_id: range.group_id,
+        start_id: range.start_id,
+        end_id: range.end_id,
+        next_id: range.next_id,
+        status: 'leased',
+        priority: range.priority ?? 0,
+        node_id: range.node_id,
+        leased_at: range.leased_at ?? new Date(),
+        heartbeat_at: new Date(),
+      })
+      .returning('id')
+    await trx.from('ranges').where('id', range.id).update({ block_id: block.id })
+    return Number(block.id)
+  }
+
   /**
-   * Release aggregates in the catalog, one write-once chunk per release, and
-   * probe candidates (a batch's, or files the fragment merger completed).
-   * Releases are sorted by key, so concurrent writers lock them in the same order.
+   * The unnamed releases of a batch into pending_releases (migration 6300, the HDD): keys, counts, the upload
+   * tool and the probe targets, no chunk. The pending ladder (REBUILD.md decision f) probes them cheapest first,
+   * and a name moves one into the catalog; a provisional tail release sent again adds its counts up.
+   */
+  async writePending(
+    trx: TransactionClientContract,
+    groupId: number,
+    releases: ReleaseChunk[],
+    candidates: ProbeCandidate[]
+  ): Promise<number> {
+    if (releases.length === 0) return 0
+    const targets = new Map<
+      string,
+      { kind: string; messageId: string; file: string; bytes?: number }[]
+    >()
+    for (const c of candidates) {
+      const key = slot(c.releaseKey, c.upload ?? 0)
+      const list = targets.get(key) ?? []
+      list.push({ kind: c.kind, messageId: c.messageId, file: c.file, bytes: c.bytes })
+      targets.set(key, list)
+    }
+    // In the record's lock order (LOG.md (136)): all rows sorted, then all classes sorted, in one transaction. The
+    // upsert locks a finished row too, before its WHERE passes it by; a chunk's classes before the next chunk's rows
+    // crossed the ladder's record(), which holds its rows and then takes its classes.
+    const rows = lockOrder(
+      releases.map((r) => {
+        const probes = targets.get(slot(r.key, r.upload)) ?? []
+        return {
+          release_key: releaseKey(r.key, groupId, r.firstArticle).toString('hex'),
+          identity_key: identityKey(r.key).toString('hex'),
+          name: r.name.slice(0, TEXT_MAX),
+          poster: r.poster || null,
+          first: r.postedAt,
+          last: r.postedLast ?? r.postedAt,
+          first_article: r.firstArticle,
+          last_article: r.lastArticle,
+          files: r.files,
+          segments: r.segments,
+          bytes: r.bytes,
+          partial: r.partial === true,
+          finished: r.finished !== false,
+          file_total: r.fileTotal,
+          tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
+          candidates: probes,
+          // A finished release without a target is through the ladder before it starts: exhausted, not waiting
+          // (1.57M rows waited forever, never claimed, before the edge gave the hash-named pieces one; LOG.md (140)).
+          status: r.finished !== false && probes.length === 0 ? PENDING.exhausted : PENDING.waiting,
+          // The first target's article (by effort): the cheapest probe first within a class (LOG.md (130)).
+          target_bytes: byEffort(probes)[0]?.bytes ?? null,
+        }
+      }),
+      (r) => r.release_key
+    )
+    let written = 0
+    const classes = new Map<string, { tool: string; klass: number; waiting: number }>()
+    for (let i = 0; i < rows.length; i += 1000) {
+      const { rows: done } = await trx.rawQuery(
+        `INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
+                                       files, segments, bytes, partial, finished, file_total, tool, candidates, klass, target_bytes, status)
+         SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
+                v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates,
+                pending_class(v.name, v.bytes, v.partial), v.target_bytes, v.status
+         FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
+                                               files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb, target_bytes bigint, status int)
+         ON CONFLICT (release_key) DO UPDATE SET
+           files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
+           last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
+           partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
+           candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END,
+           klass = pending_class(pending_releases.name, pending_releases.bytes + excluded.bytes, excluded.partial),
+           target_bytes = coalesce(pending_releases.target_bytes, excluded.target_bytes),
+           status = CASE WHEN pending_releases.status = ? AND excluded.finished
+                              AND jsonb_array_length(pending_releases.candidates) = 0 AND jsonb_array_length(excluded.candidates) = 0
+                         THEN ? ELSE pending_releases.status END
+         WHERE NOT pending_releases.finished
+         RETURNING release_key, tool, klass, (xmax = 0) AS inserted, status`,
+        [groupId, JSON.stringify(rows.slice(i, i + 1000)), PENDING.waiting, PENDING.exhausted]
+      )
+      written += done.length
+      // Each class seen gets its row in the ladder's record, so the claim explores it (pending_classes).
+      for (const d of done) {
+        const key = `${d.tool}\t${d.klass}`
+        const entry = classes.get(key) ?? { tool: d.tool, klass: Number(d.klass), waiting: 0 }
+        if (d.inserted && Number(d.status) === PENDING.waiting) entry.waiting++
+        classes.set(key, entry)
+      }
+    }
+    if (classes.size > 0) {
+      await trx.rawQuery(
+        `INSERT INTO pending_classes (tool, group_id, klass, waiting)
+         SELECT v.tool, ?, v.klass, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, klass int, waiting int)
+         ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = pending_classes.waiting + excluded.waiting`,
+        [
+          groupId,
+          JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${groupId}\t${c.klass}`)),
+        ]
+      )
+    }
+    return written
+  }
+
+  /**
+   * The releases of a batch into the catalog, one chunk each, and their probe candidates. Every chunk is one
+   * release: inserted, or (a provisional tail release sent again) finished.
    */
   async writeReleases(
     trx: TransactionClientContract,
-    rangeId: number,
+    where: { groupId: number; blockId: number },
     batchFrom: number,
     releases: ReleaseChunk[],
     candidates: ProbeCandidate[]
   ) {
-    const out = { newReleases: 0, candidates: 0, ids: new Map<string, number>() }
-    const ids = out.ids
-    // Releases, sorted by key so concurrent batches lock them in the same order.
-    const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key))
+    const out = { newReleases: 0, finalized: 0, candidates: 0 }
+    if (releases.length === 0) return out
+    /** Release ids by slot, for the chunks and candidates; a slot not here was a finished release sent again. */
+    const ids = new Map<string, number>()
+    const finalized = new Set<number>()
+
+    const posters = await CatalogRefs.posterIds(trx, releases.map((r) => r.poster).filter(Boolean))
+    const groupSets = await CatalogRefs.groupSetIds(
+      trx,
+      releases.map((r) => r.groups)
+    )
+    const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
     for (let i = 0; i < sorted.length; i += 1000) {
-      const chunk = sorted.slice(i, i + 1000)
-      const { rows } = await trx.rawQuery(
-        `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2)
-           SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2
-           FROM jsonb_to_recordset(?::jsonb) AS v(key text, name text, source text, poster text, groups jsonb, posted_at bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
-           ON CONFLICT (release_key) DO UPDATE SET
-             files = releases.files + excluded.files,
-             segments = releases.segments + excluded.segments,
-             bytes = releases.bytes + excluded.bytes,
-             chunks = releases.chunks + 1,
-             posted_at = least(releases.posted_at, excluded.posted_at),
-             file_total = coalesce(releases.file_total, excluded.file_total),
-             first_article = least(releases.first_article, excluded.first_article),
-             last_article = greatest(releases.last_article, excluded.last_article),
-             readable = releases.readable OR excluded.readable,
-             has_par2 = releases.has_par2 OR excluded.has_par2,
-             last_seen_at = now()
-           RETURNING id, release_key, (xmax = 0) AS inserted`,
-        [
-          JSON.stringify(
-            chunk.map((r) => ({
-              key: r.key,
-              name: r.name,
-              source: r.source,
-              poster: r.poster,
-              groups: r.groups,
-              posted_at: r.postedAt,
-              file_total: r.fileTotal,
-              files: r.files,
-              segments: r.segments,
-              bytes: r.bytes,
-              first_article: r.firstArticle,
-              last_article: r.lastArticle,
-              readable: r.readable,
-              has_par2: r.hasPar2,
-            }))
-          ),
-        ]
+      const part = sorted.slice(i, i + 1000)
+      const rows = part.map((r, idx) => ({
+        idx,
+        release_key: releaseKey(r.key, where.groupId, r.firstArticle).toString('hex'),
+        identity_key: identityKey(r.key).toString('hex'),
+        name: r.name,
+        source: sourceCode(r.source),
+        poster_id: r.poster ? (posters.get(posterText(r.poster)) ?? null) : null,
+        group_set_id: groupSets.get(groupSetKey(r.groups))!,
+        first: r.postedAt,
+        last: r.postedLast ?? r.postedAt,
+        file_total: r.fileTotal,
+        files: r.files,
+        segments: r.segments,
+        bytes: r.bytes,
+        first_article: r.firstArticle,
+        last_article: r.lastArticle,
+        readable: r.readable,
+        has_par2: r.hasPar2,
+        finished: r.finished !== false,
+        edge: r.edge === true,
+        partial: r.partial === true,
+        sidecar_only: sidecarOnly(r),
+      }))
+      // Inserted, or a provisional release of the key finished (the chunk's files added up); a finished release
+      // sent again (no row returned) is left as it is.
+      const { rows: written } = await trx.rawQuery(
+        `INSERT INTO releases (release_key, identity_key, group_id, group_set_id, poster_id, name, source, posted_at, last_posted_at, file_total, files, segments, bytes, chunks,
+                               first_article, last_article, readable, has_par2, complete, edge, sidecar_only, settled_at)
+         SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.group_set_id, v.poster_id, v.name, v.source,
+                to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
+                v.first_article, v.last_article, v.readable, v.has_par2,
+                CASE WHEN v.partial THEN false WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END,
+                v.edge, v.sidecar_only, CASE WHEN v.finished THEN now() END
+         FROM jsonb_to_recordset(?::jsonb) AS v(idx int, release_key text, identity_key text, name text, source int, poster_id int, group_set_id int, first bigint, last bigint,
+                                               file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean,
+                                               finished boolean, edge boolean, partial boolean, sidecar_only boolean)
+         ON CONFLICT (release_key) DO UPDATE SET
+           files = releases.files + excluded.files, segments = releases.segments + excluded.segments, bytes = releases.bytes + excluded.bytes,
+           chunks = releases.chunks + CASE WHEN excluded.files > 0 THEN 1 ELSE 0 END,
+           posted_at = least(releases.posted_at, excluded.posted_at), last_posted_at = greatest(releases.last_posted_at, excluded.last_posted_at),
+           file_total = coalesce(excluded.file_total, releases.file_total),
+           first_article = least(releases.first_article, excluded.first_article), last_article = greatest(releases.last_article, excluded.last_article),
+           readable = releases.readable OR excluded.readable, has_par2 = releases.has_par2 OR excluded.has_par2,
+           complete = CASE WHEN releases.complete IS FALSE AND releases.file_total IS NULL THEN false
+                           WHEN coalesce(excluded.file_total, releases.file_total) IS NULL THEN releases.complete
+                           ELSE releases.files + excluded.files >= coalesce(excluded.file_total, releases.file_total) END,
+           edge = excluded.edge, sidecar_only = coalesce(releases.sidecar_only, excluded.sidecar_only), settled_at = excluded.settled_at
+         WHERE releases.settled_at IS NULL
+         RETURNING id, encode(release_key, 'hex') AS release_key, (xmax = 0) AS inserted, settled_at`,
+        [where.groupId, JSON.stringify(rows)]
       )
+      const byKey = new Map<string, { id: number; inserted: boolean; settled: boolean }>(
+        written.map(
+          (w: { id: string; release_key: string; inserted: boolean; settled_at: Date | null }) => [
+            w.release_key,
+            { id: Number(w.id), inserted: w.inserted, settled: w.settled_at !== null },
+          ]
+        )
+      )
+      const texts: { release_id: number; kind: number; text: string }[] = []
       for (const row of rows) {
-        ids.set(row.release_key, Number(row.id))
-        if (row.inserted) out.newReleases++
+        const w = byKey.get(row.release_key)
+        if (!w) continue
+        const r = part[row.idx]
+        ids.set(slot(r.key, r.upload), w.id)
+        if (w.inserted) out.newReleases++
+        else if (w.settled) {
+          out.finalized++
+          finalized.add(w.id)
+        }
+        texts.push({ release_id: w.id, kind: TEXT_KINDS.name, text: row.name.slice(0, TEXT_MAX) })
+        // An album's members stay findable by their own names (the first TEXTS_PER_KIND of them).
+        if (r.source === 'album' && r.files > 0) {
+          for (const text of textsToStore(decodeChunk(r.payload).map((f) => f.name))) {
+            texts.push({ release_id: w.id, kind: TEXT_KINDS.member, text })
+          }
+        }
+      }
+      // The name (and an album's members), searchable (release_texts; a text sent again is a no-op).
+      if (texts.length > 0) {
+        await trx.rawQuery(
+          `INSERT INTO release_texts (release_id, kind, text)
+           SELECT v.release_id, v.kind, v.text FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, kind int, text text) WHERE v.text <> ''
+           ON CONFLICT DO NOTHING`,
+          [JSON.stringify(texts)]
+        )
       }
     }
 
-    // One write-once chunk per release.
+    // One write-once chunk per release (a finish adds the files not sent before).
     for (let i = 0; i < sorted.length; i += 500) {
-      const rows = sorted.slice(i, i + 500).map((r) => ({
-        release_id: ids.get(r.key)!,
-        range_id: rangeId,
-        batch_from: batchFrom,
-        files: r.files,
-        segments: r.segments,
-        payload: r.payload,
-      }))
+      const rows = sorted
+        .slice(i, i + 500)
+        .filter((r) => ids.has(slot(r.key, r.upload)) && r.files > 0)
+        .map((r) => ({
+          release_id: ids.get(slot(r.key, r.upload))!,
+          block_id: where.blockId,
+          batch_from: batchFrom,
+          files: r.files,
+          segments: r.segments,
+          payload: r.payload,
+        }))
       if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
     }
 
-    const kept = candidates.filter((c) => ids.has(c.releaseKey))
+    // A finished release's candidates are claimable at once; a provisional one's wait for its finish.
+    const settled = new Set<number>()
+    for (const r of sorted) {
+      const id = ids.get(slot(r.key, r.upload))
+      if (id !== undefined && r.finished !== false) settled.add(id)
+    }
+    const chunks = new Map(sorted.map((r) => [slot(r.key, r.upload), r]))
+    const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
     for (let i = 0; i < kept.length; i += 2000) {
-      const rows = kept
-        .slice(i, i + 2000)
-        .map((c) => ({
-          release_id: ids.get(c.releaseKey)!,
+      const rows = kept.slice(i, i + 2000).map((c) => {
+        const releaseId = ids.get(slot(c.releaseKey, c.upload ?? 0))!
+        const chunk = chunks.get(slot(c.releaseKey, c.upload ?? 0))!
+        return {
+          release_id: releaseId,
           kind: c.kind,
           message_id: c.messageId,
           file: c.file,
-        }))
+          ready: settled.has(releaseId),
+          unreadable: !chunk.readable,
+        }
+      })
       const added = await trx
         .knexQuery()
         .table('probe_candidates')
@@ -185,6 +429,17 @@ class BatchStore {
         .ignore()
         .returning('id')
       out.candidates += added.length
+      // A later piece of a split 7z: the tails of lower volumes go (app/services/seven_zip_tails.ts).
+      const tails = new Set(rows.filter((r) => r.kind === '7z-tail').map((r) => r.release_id))
+      if (added.length > 0 && tails.size > 0) await pruneSevenZipTails(trx, [...tails])
+    }
+    // The finish of a provisional release: its waiting candidates become claimable.
+    if (finalized.size > 0) {
+      await trx.rawQuery(
+        `UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
+         WHERE pc.release_id = ANY(?) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
+        [[...finalized]]
+      )
     }
 
     return out

app/services/catalog_refs.ts

@@ -0,0 +1,93 @@
+import type db from '@adonisjs/lucid/services/db'
+
+type Client = Pick<typeof db, 'rawQuery'>
+
+/** Posters are kept to this length (as the trigram index and poster search always did; a btree entry holds 2,704 bytes). */
+export const POSTER_MAX = 200
+/** Cached ids per process: group sets are ~2,600 in all; posters many more, so the cache is emptied when it grows past this. */
+const CACHE_MAX = 200_000
+
+const posterCache = new Map<string, number>()
+const groupSetCache = new Map<string, number>()
+
+/** A group list as group_sets stores it: sorted, without repeats, as JSON. */
+export function groupSetKey(groups: string[]): string {
+  return JSON.stringify([...new Set(groups)].sort())
+}
+
+/** A poster as posters stores it. */
+export function posterText(poster: string): string {
+  return poster.slice(0, POSTER_MAX)
+}
+
+async function ids(
+  client: Client,
+  cache: Map<string, number>,
+  table: 'posters' | 'group_sets',
+  column: 'poster' | 'groups',
+  cast: 'text' | 'jsonb',
+  values: string[]
+): Promise<Map<string, number>> {
+  const out = new Map<string, number>()
+  const missing: string[] = []
+  for (const v of values) {
+    const id = cache.get(v)
+    if (id === undefined) missing.push(v)
+    else out.set(v, id)
+  }
+  if (missing.length === 0) return out
+  // Insert what's new, then read every id (ON CONFLICT DO NOTHING returns nothing for the rows that existed).
+  await client.rawQuery(
+    `INSERT INTO ${table} (${column}) SELECT DISTINCT v::${cast} FROM unnest(?::text[]) AS v ON CONFLICT DO NOTHING`,
+    [missing]
+  )
+  const { rows } = await client.rawQuery(
+    `SELECT id, ${column}::text AS value FROM ${table} WHERE ${column} = ANY(?::${cast}[])`,
+    [missing]
+  )
+  if (cache.size + rows.length > CACHE_MAX) cache.clear()
+  for (const r of rows) {
+    // jsonb comes back in Postgres' spelling ("["a", "b"]"); the caller's key is the JSON we sent.
+    const key = cast === 'jsonb' ? JSON.stringify(JSON.parse(r.value)) : r.value
+    cache.set(key, Number(r.id))
+    out.set(key, Number(r.id))
+  }
+  return out
+}
+
+/**
+ * Ids in the catalog's lookup tables (voyager 3, migration 6000): a release's poster_id and group_set_id. New
+ * values are inserted; known ones come from a cache per process (the batch store sees the same posters and
+ * group sets batch after batch).
+ */
+export default {
+  /** Ids by poster (posterText applied), for a batch of posters. */
+  async posterIds(client: Client, posters: string[]): Promise<Map<string, number>> {
+    return ids(client, posterCache, 'posters', 'poster', 'text', [
+      ...new Set(posters.map(posterText)),
+    ])
+  },
+
+  async posterId(client: Client, poster: string): Promise<number> {
+    const found = await this.posterIds(client, [poster])
+    return found.get(posterText(poster))!
+  },
+
+  /** Ids by groupSetKey, for a batch of group lists. */
+  async groupSetIds(client: Client, sets: string[][]): Promise<Map<string, number>> {
+    return ids(client, groupSetCache, 'group_sets', 'groups', 'jsonb', [
+      ...new Set(sets.map(groupSetKey)),
+    ])
+  },
+
+  async groupSetId(client: Client, groups: string[]): Promise<number> {
+    const found = await this.groupSetIds(client, [groups])
+    return found.get(groupSetKey(groups))!
+  },
+
+  /** Forgets the cached ids (tests roll their transactions back, so the ids they saw are gone). */
+  forget() {
+    posterCache.clear()
+    groupSetCache.clear()
+  },
+}

app/services/catalog_space.ts

@@ -0,0 +1,38 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import coordinatorConfig from '#config/coordinator'
+
+/** How often the catalog's size is measured (pg_tablespace_size walks its files). */
+const CHECK_MS = 60_000
+
+/**
+ * The catalog's room on the NVMe (CATALOG_MAX_NVME_GB, like voyager 1's
+ * BACKFILL_MAX_NVME_GB): once its tablespace (the database's
+ * default_tablespace) is over the cap, only the tail and the stored ranges
+ * (priority below 0) are leased and prefetched, and history waits. After
+ * reset 24 the catalog grew ~3.7 GB an hour, root down to 31 GB free with
+ * 600k history ranges to go (LOG.md 2026-10-14 (53)). Off when unset.
+ */
+class CatalogSpace {
+  private checkedAt = 0
+  private over = false
+
+  async full(): Promise<boolean> {
+    const cap = coordinatorConfig.catalogMaxNvmeGb
+    if (!cap) return false
+    if (Date.now() - this.checkedAt < CHECK_MS) return this.over
+    this.checkedAt = Date.now()
+    const { rows } = await db.rawQuery(
+      `SELECT pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
+    )
+    const bytes = Number(rows[0].bytes)
+    const was = this.over
+    this.over = bytes > cap * 1e9
+    if (this.over !== was) {
+      logger.info(`Catalog ${(bytes / 1e9).toFixed(1)} GB of ${cap}: history ${this.over ? 'paused (tail and stored ranges only)' : 'resumed'}.`)
+    }
+    return this.over
+  }
+}
+
+export default new CatalogSpace()

app/services/corpus.ts

@@ -0,0 +1,120 @@
+import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { deflate as deflateCb, gzip as gzipCb, unzip } from 'node:zlib'
+import { promisify } from 'node:util'
+import coordinatorConfig from '#config/coordinator'
+import { filterOverviews } from '#lib/edge/index'
+import type { ChunkSpan } from '#services/spool'
+
+const inflate = promisify(unzip)
+const deflate = promisify(deflateCb)
+const gzip = promisify(gzipCb)
+/** The largest inflated chunk the filter reads as one string (V8's limit is 0x1fffffe8 characters). */
+const MAX_FILTER_BYTES = 256 * 1024 * 1024
+
+/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
+export interface CorpusFilter {
+  ruleVersion: number
+  priorities: Record<string, number>
+}
+/** The rules a filtered chunk leaves out (lib/edge/corpus_filter.ts), in its range.json. */
+export const FILTERED = ['crosspost', 'obfuscated']
+
+export interface CorpusRange {
+  group: string
+  start: number
+  end: number
+}
+
+/**
+ * Stored XOVER chunks of the ranges marked `capture` (REBUILD.md, corpus):
+ * `<dir>/<group>/<start>-<end>/<nnnn>-<from>-<to>.gz` with a range.json once
+ * every chunk is there, the layout edge-bench reads (edge/fixtures). The
+ * grouping sweeps replay them with each build's rules instead of waiting for
+ * a re-index; the prefetcher reads a stored chunk instead of fetching it
+ * again. Off without CORPUS_DIR. Filtered (CORPUS_FILTER; LOG.md 2026-10-23
+ * (101)): the lines the edge worker only counts are left out, ~half of them,
+ * and each chunk's `.drops` beside it (gzip JSON, ChunkDrops) lists their
+ * article numbers by rule; its range.json says `filtered`.
+ */
+class Corpus {
+  get dir(): string | undefined {
+    return coordinatorConfig.corpus.dir
+  }
+
+  private rangeDir(range: CorpusRange, dir = this.dir!) {
+    return join(dir, range.group, `${range.start}-${range.end}`)
+  }
+
+  private path(range: CorpusRange, span: ChunkSpan, dir = this.dir!) {
+    const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
+    return join(this.rangeDir(range, dir), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
+  }
+
+  /** Whether every chunk of the range is stored (its range.json). */
+  async complete(range: CorpusRange): Promise<boolean> {
+    if (!this.dir) return false
+    return access(join(this.rangeDir(range), 'range.json')).then(
+      () => true,
+      () => false
+    )
+  }
+
+  /** Whether one chunk is stored. */
+  async has(range: CorpusRange, span: ChunkSpan): Promise<boolean> {
+    if (!this.dir) return false
+    return access(this.path(range, span)).then(
+      () => true,
+      () => false
+    )
+  }
+
+  /** A stored chunk, or null (`dir`: another corpus than CORPUS_DIR's, the namer's tests). */
+  async read(range: CorpusRange, span: ChunkSpan, dir = this.dir): Promise<Buffer | null> {
+    if (!dir) return null
+    return readFile(this.path(range, span, dir)).catch(() => null)
+  }
+
+  /**
+   * Stores a chunk that inflates (a reply cut short isn't kept), and the
+   * range.json once the range is complete (`total` chunks). With `filter`,
+   * without the lines the edge worker drops on their own, compressed again
+   * as the news server's (zlib), with its `.drops` written first.
+   */
+  async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number, filter?: CorpusFilter) {
+    if (!this.dir) return
+    let raw: Buffer
+    try {
+      raw = await inflate(data)
+    } catch {
+      return
+    }
+    const dir = this.rangeDir(range)
+    const file = this.path(range, span)
+    await mkdir(dir, { recursive: true })
+    let stored = data
+    // Past V8's longest string (0x1fffffe8) the text can't be filtered; such a chunk is stored as it came (the
+    // worker drops the same lines itself). Seen once: a chunk inflating past 512 MB (LOG.md 2026-10-24 (102)).
+    if (filter && raw.length < MAX_FILTER_BYTES) {
+      // Latin-1 both ways: every byte back as it was.
+      const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
+      stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
+      await writeFile(`${file}.drops.part`, await gzip(JSON.stringify(drops)))
+      await rename(`${file}.drops.part`, `${file}.drops`)
+    }
+    await writeFile(`${file}.part`, stored)
+    await rename(`${file}.part`, file)
+    const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
+    if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(filter ? { ...range, filtered: FILTERED } : range))
+  }
+
+  /** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
+  async removeChunk(range: CorpusRange, span: ChunkSpan) {
+    if (!this.dir) return
+    await rm(this.path(range, span), { force: true })
+    await rm(`${this.path(range, span)}.drops`, { force: true })
+    await rm(join(this.rangeDir(range), 'range.json'), { force: true })
+  }
+}
+
+export default new Corpus()

app/services/disk_space.ts

@@ -0,0 +1,40 @@
+import { statfs } from 'node:fs/promises'
+import logger from '@adonisjs/core/services/logger'
+
+/** How often free space is measured. */
+const CHECK_MS = 60_000
+/** Resumed only this much above the floor: not paused and resumed at every flush. */
+const MARGIN_GB = 1
+
+/**
+ * Free space on the filesystem holding `path`, against a floor: low below it, again fine MARGIN_GB above it. In a
+ * container `/` is an overlay on the host's Docker root, which on homelab is the NVMe the catalog and the WAL are on.
+ * The prober filled it ~0.8–1.1 GB an hour at ~200 articles/s, and nothing stopped it at the 10 GB floor (LOG.md
+ * 2026-10-18 (77)). Off when the floor is 0.
+ */
+export class DiskSpace {
+  private checkedAt = 0
+  private low = false
+
+  constructor(
+    private what: string,
+    private path: string,
+    private floorGb: number,
+    private free: (path: string) => Promise<number> = async (p) => {
+      const s = await statfs(p)
+      return s.bavail * s.bsize
+    },
+    private checkMs = CHECK_MS
+  ) {}
+
+  async short(): Promise<boolean> {
+    if (!this.floorGb) return false
+    if (Date.now() - this.checkedAt < this.checkMs) return this.low
+    this.checkedAt = Date.now()
+    const gb = (await this.free(this.path)) / 1e9
+    const was = this.low
+    this.low = gb < this.floorGb + (was ? MARGIN_GB : 0)
+    if (this.low !== was) logger.info(`${gb.toFixed(1)} GB free (floor ${this.floorGb} GB): ${this.what} ${this.low ? 'paused' : 'resumed'}.`)
+    return this.low
+  }
+}

app/services/fragment_merger.ts

@@ -1,151 +0,0 @@
-import db from '@adonisjs/lucid/services/db'
-import BatchStore from '#services/batch_store'
-import ChunkPartitions from '#services/chunk_partitions'
-import RuleSets from '#services/rule_sets'
-import { RuleSet } from '#lib/rules/index'
-import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
-import type { AggregatedFile } from '#lib/files/index'
-import type { EdgeSegment } from '#lib/edge/index'
-
-export interface MergeResult {
-  files: number
-  completed: number
-  incomplete: number
-  waiting: number
-  /** This round reached the end of the file keys (the next one starts over). */
-  wrapped: boolean
-}
-
-/** File keys per round. */
-const BATCH = 20_000
-
-/**
- * Files cut by a range's edge (REBUILD.md, fragments), put together once
- * the ranges around them are done: a complete file is judged by the rules
- * like any other and written as a release chunk (with probe candidates); a
- * file still missing parts while every neighbouring range is done is
- * incomplete, and dropped (counted on its ranges). Others wait.
- */
-class FragmentMerger {
-  /** Where the next round goes on: waiting files must not take every round's places. */
-  private cursor = ''
-
-  async run(): Promise<MergeResult> {
-    const result: MergeResult = { files: 0, completed: 0, incomplete: 0, waiting: 0, wrapped: false }
-    const { rows: keys } = await db.rawQuery(
-      `SELECT DISTINCT f.file_key FROM fragments f JOIN ranges r ON r.id = f.range_id
-       WHERE r.status = 'done' AND f.file_key > ? ORDER BY f.file_key LIMIT ?`,
-      [this.cursor, BATCH]
-    )
-    // At the end of the keys: the next round starts over.
-    result.wrapped = keys.length < BATCH
-    this.cursor = result.wrapped ? '' : keys.at(-1).file_key
-    if (keys.length === 0) return result
-    const rules = new RuleSet(await RuleSets.current())
-    const rows = await db
-      .from('fragments as f')
-      .join('ranges as r', 'r.id', 'f.range_id')
-      .whereIn('f.file_key', keys.map((k: { file_key: string }) => k.file_key))
-      .select('f.id', 'f.file_key', 'f.range_id', 'f.fragment', 'r.group_id', 'r.start_id', 'r.end_id', 'r.status')
-    const byKey = new Map<string, typeof rows>()
-    for (const row of rows) {
-      const list = byKey.get(row.file_key)
-      if (list) list.push(row)
-      else byKey.set(row.file_key, [row])
-    }
-
-    const completed: { file: AggregatedFile<EdgeSegment>; rangeId: number; ids: number[] }[] = []
-    const incomplete: { rangeId: number; ids: number[]; segments: number }[] = []
-    const neighbours = await this.neighbourStatus([...byKey.values()])
-    for (const list of byKey.values()) {
-      result.files++
-      const file = mergeFragments(list.map((r) => r.fragment as Fragment))
-      const ids = list.map((r) => Number(r.id))
-      const rangeId = Math.max(...list.map((r) => Number(r.range_id)))
-      if (file) {
-        completed.push({ file, rangeId, ids })
-        continue
-      }
-      if (list.every((r) => r.status === 'done') && neighbours(list)) {
-        incomplete.push({ rangeId, ids, segments: list.reduce((n, r) => n + (r.fragment as Fragment).segments.length, 0) })
-      } else {
-        result.waiting++
-      }
-    }
-
-    // Completed files, grouped into their releases, judged and written like a batch's.
-    const byRelease = new Map<string, { files: AggregatedFile<EdgeSegment>[]; rangeId: number }>()
-    for (const { file, rangeId } of completed) {
-      const key = releaseKeyOf(file.segments[0])
-      const entry = byRelease.get(key)
-      if (entry) entry.files.push(file)
-      else byRelease.set(key, { files: [file], rangeId })
-    }
-    const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
-    for (const [key, { files, rangeId }] of byRelease) {
-      const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId })))
-      if (verdict.rule !== null) continue
-      releases.push({ rangeId, chunk: chunkOf(key, files), candidates: candidatesOf(key, files) })
-    }
-    await ChunkPartitions.ensure()
-    await db.transaction(async (trx) => {
-      for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
-      const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
-      for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()
-      for (const { rangeId, segments } of incomplete) {
-        await trx.rawQuery(
-          `INSERT INTO rule_counts (range_id, rule, key, articles, files) VALUES (?, 'incomplete', 'edge', ?, 1)
-           ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + 1`,
-          [rangeId, segments]
-        )
-      }
-    })
-    result.completed = completed.length
-    result.incomplete = incomplete.length
-    return result
-  }
-
-  /**
-   * For each file's fragments: whether the ranges right before and after
-   * their ranges (where a missing part could be) are done, read in one
-   * query for the whole round (per file it was three, ~2,000 files a
-   * minute). A neighbour not planned yet counts as not done (newer articles
-   * come later), except before the group's oldest article.
-   */
-  private async neighbourStatus(lists: { group_id: number; start_id: number | string; end_id: number | string }[][]) {
-    const span = (list: (typeof lists)[number]) => ({
-      group: Number(list[0].group_id),
-      first: Math.min(...list.map((r) => Number(r.start_id))),
-      last: Math.max(...list.map((r) => Number(r.end_id))),
-    })
-    const spans = lists.map(span)
-    const groups = [...new Set(spans.map((s) => s.group))]
-    // The range holding the article before and the one after, each by the (group_id, start_id) index:
-    // "end_id = ? OR start_id = ?" read every range of the group per file (5.5 s a round; LOG.md 2026-10-09).
-    const { rows } = await db.rawQuery(
-      `SELECT s.group_id, s.before_end, s.after_start, b.status AS before, a.status AS after
-       FROM unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start)
-       LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.before_end AND r.end_id >= s.before_end
-                          ORDER BY r.start_id DESC LIMIT 1) b ON true
-       LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.after_start AND r.end_id >= s.after_start
-                          ORDER BY r.start_id DESC LIMIT 1) a ON true`,
-      [spans.map((s) => s.group), spans.map((s) => s.first - 1), spans.map((s) => s.last + 1)]
-    )
-    const byEnd = new Map<string, string>()
-    const byStart = new Map<string, string>()
-    for (const r of rows) {
-      if (r.before !== null) byEnd.set(`${r.group_id}:${r.before_end}`, r.before)
-      if (r.after !== null) byStart.set(`${r.group_id}:${r.after_start}`, r.after)
-    }
-    const lows = new Map((await db.from('groups').whereIn('id', groups).select('id', 'low')).map((g) => [Number(g.id), g.low === null ? null : Number(g.low)]))
-    return (list: (typeof lists)[number]) => {
-      const { group, first, last } = span(list)
-      const before = byEnd.get(`${group}:${first - 1}`)
-      const low = lows.get(group)
-      const beforeDone = before !== undefined ? before === 'done' : low === null || low === undefined || first <= low
-      return beforeDone && byStart.get(`${group}:${last + 1}`) === 'done'
-    }
-  }
-}
-
-export default new FragmentMerger()

app/services/group_dates.ts

@@ -0,0 +1,138 @@
+import type { Overview, XoverClient } from '#lib/nntp/index'
+
+/** When an article of a group was posted: the median Date of a slice of articles at `article`. */
+export interface DateSample {
+  article: number
+  postedAt: Date
+}
+
+/** Before this, a Date header is a broken clock (the oldest retention is 2008). */
+const OLDEST = Date.UTC(1995, 0, 1)
+const DAY = 86_400_000
+
+/**
+ * The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least
+ * `min` of them; null otherwise (an expired or empty slice).
+ */
+export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null {
+  const latest = now.getTime() + DAY
+  const times = overviews
+    .map(([, fields]) => Date.parse(fields.date ?? ''))
+    .filter((t) => t >= OLDEST && t <= latest)
+    .sort((a, b) => a - b)
+  if (times.length < min) return null
+  return new Date(times[Math.floor(times.length / 2)])
+}
+
+/** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */
+export function samplePoints(low: number, high: number, count: number, size: number): number[] {
+  const last = Math.max(low, high - size + 1)
+  if (count <= 1 || last <= low) return [low]
+  const points = new Set<number>()
+  for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1)))
+  return [...points].sort((a, b) => a - b)
+}
+
+/**
+ * The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day
+ * (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive
+ * in number order, bar stragglers and old crossposts).
+ */
+export function monotone(samples: DateSample[]): DateSample[] {
+  const sorted = [...samples].sort((a, b) => a.article - b.article)
+  const kept = sorted.filter((s, i) => {
+    const next = sorted.slice(i + 1, i + 3)
+    return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime())
+  })
+  let floor = Number.NEGATIVE_INFINITY
+  return kept.map((s) => {
+    floor = Math.max(floor, s.postedAt.getTime())
+    return { article: s.article, postedAt: new Date(floor) }
+  })
+}
+
+/**
+ * When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around
+ * it, the first or last sample's date beyond them; null without samples.
+ */
+export function estimateDate(samples: DateSample[], article: number): Date | null {
+  if (samples.length === 0) return null
+  if (article <= samples[0].article) return samples[0].postedAt
+  const last = samples[samples.length - 1]
+  if (article >= last.article) return last.postedAt
+  let lo = 0
+  let hi = samples.length - 1
+  while (hi - lo > 1) {
+    const mid = (lo + hi) >> 1
+    if (samples[mid].article <= article) lo = mid
+    else hi = mid
+  }
+  const a = samples[lo]
+  const b = samples[hi]
+  const f = (article - a.article) / (b.article - a.article)
+  return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime()))
+}
+
+/** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */
+export function sampleCount(
+  low: number,
+  high: number,
+  every: number,
+  min = 16,
+  max = 1000
+): number {
+  return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1))
+}
+
+/**
+ * Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice
+ * with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the
+ * server's span.
+ */
+export async function sampleGroup(
+  use: <T>(work: (client: XoverClient) => Promise<T>) => Promise<T>,
+  name: string,
+  options: { every: number; size: number; maxFailures?: number },
+  now = new Date()
+): Promise<{
+  samples: DateSample[]
+  low: number
+  high: number
+  commands: number
+  failed: string[]
+}> {
+  const info = await use((client) => client.group(name))
+  const samples: DateSample[] = []
+  const failed: string[] = []
+  let commands = 1
+  // Each point on a connection of its own (GROUP again): a reply that times out (Eweka stalls some requests for
+  // minutes; 16 groups never got through on one connection) costs that point, not the group.
+  for (const point of samplePoints(
+    info.low,
+    info.high,
+    sampleCount(info.low, info.high, options.every),
+    options.size
+  )) {
+    if (failed.length >= (options.maxFailures ?? 3)) break
+    try {
+      const date = await use(async (client) => {
+        await client.group(name)
+        commands += 2
+        const slice = medianDate(
+          await client.xover(point, Math.min(point + options.size - 1, info.high)),
+          now
+        )
+        if (slice) return slice
+        commands++
+        return medianDate(
+          await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)),
+          now
+        )
+      })
+      if (date) samples.push({ article: point, postedAt: date })
+    } catch (error: any) {
+      failed.push(`${point}: ${error.message}`)
+    }
+  }
+  return { samples: monotone(samples), low: info.low, high: info.high, commands, failed }
+}

app/services/group_leases.ts

@@ -0,0 +1,280 @@
+import db from '@adonisjs/lucid/services/db'
+import { createReadStream, createWriteStream } from 'node:fs'
+import { mkdir, rename, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import coordinatorConfig from '#config/coordinator'
+import RuleSets from '#services/rule_sets'
+import CatalogSpace from '#services/catalog_space'
+import { LeaseLost } from '#services/leases'
+import type { GroupLease, Lane, NextRange } from '#lib/edge/index'
+
+export const LANES: Lane[] = ['history', 'tail']
+
+/**
+ * Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and asks for its
+ * next range each time it finished one, so a group's ranges are indexed in article order through one carried
+ * window. The coordinator keeps one row a group and lane (group_leases): the holder, its heartbeat, the range it
+ * works and the snapshot its window stands at.
+ *
+ * - Two lanes (2026-10-27): the `history` lane works the ranges the bounds hand out (LEASE_MIN/MAX_PRIORITY,
+ *   LEASE_CAPTURE_ONLY: the windows, the history), the `tail` lane the tail planner's ranges (priority at or below
+ *   the tail's). Each lane has its own holder, window and snapshot, so a thread on the tail never wades through a
+ *   group's history first (a group's ranges come in article order, and a held group's history is days of ranges), and
+ *   the tail of a group whose history is held isn't stuck behind it. The one seam a group is where the history lane
+ *   reaches the tail lane's first range (its ranges are done: the history lane is caught up).
+ * - lease: the free group with the most urgent planned range of the lane (the ranges' priority), and among those a
+ *   mix: the group with the most planned articles and the longest unheld one (never held first) take turns, so a
+ *   big group snapshotted at a stop doesn't wait behind hundreds of small never-held ones, and a quiet group's tail
+ *   is visited all the same. A holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group resets what the
+ *   lane's snapshot doesn't cover: the lane's ranges from the snapshot's article on that the window did after it
+ *   (leased, done since the snapshot, or read past it) are planned again (the batch store's keys make the redo
+ *   idempotent); without a snapshot the window is lost and the group continues at its resume point (one seam).
+ * - next: the group's lowest planned range of the lane (or the one this node already has), leased to the node; none
+ *   when the lane is caught up: the worker snapshots and gives the group back, and gets it again when the tail
+ *   planned more.
+ * - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR; `<group>.gz`, the tail's
+ *   `<group>-tail.gz`), given to the next holder of the lane.
+ */
+class GroupLeases {
+  get dir(): string {
+    return coordinatorConfig.snapshotDir
+  }
+
+  /**
+   * The lane's ranges, as a condition on `r`: the tail lane's are at or below the tail's priority, the history lane's
+   * above it — within the lease bounds when `bounds` (what is handed out), or all of them (what a lane's window did).
+   */
+  private laneFilter(lane: Lane, bounds: boolean): { sql: string; bindings: number[] } {
+    const tail = coordinatorConfig.tail.priority
+    if (lane === 'tail') return { sql: 'r.priority <= ?', bindings: [tail] }
+    if (!bounds) return { sql: 'r.priority > ?', bindings: [tail] }
+    const { leasePriorityMin: min, leasePriorityMax: max, leaseCaptureOnly } = coordinatorConfig
+    return {
+      sql: `r.priority > ? AND r.priority BETWEEN ? AND ?${leaseCaptureOnly ? ' AND r.capture' : ''}`,
+      bindings: [tail, min, max],
+    }
+  }
+
+  async lease(nodeId: number, lane: Lane = 'history'): Promise<GroupLease | null> {
+    // The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
+    if (lane === 'history' && (await CatalogSpace.full())) return null
+    const rules = await RuleSets.current()
+    const ttl = coordinatorConfig.leaseTtlSeconds
+    const todo = this.laneFilter(lane, true)
+    const all = this.laneFilter(lane, false)
+    return db.transaction(async (trx) => {
+      // One lease at a time: the choice reads several tables.
+      await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
+      // The free groups with planned work in the lane (a range leased by a holder gone quiet counts: a group whose
+      // last range that holder took would never be taken over), by the most urgent range's priority; within a priority the
+      // most planned articles and the longest unheld alternate: one lease starts from the biggest, the next from
+      // the longest unheld (the sequence counts the leases; ranked afresh each time, "biggest first" would always
+      // win).
+      const { rows: turns } = await trx.rawQuery(`SELECT nextval('group_lease_turns') % 2 AS odd`)
+      const [first, second] = Number(turns[0].odd) === 1 ? [1, 0] : [0, 1]
+      const { rows } = await trx.rawQuery(
+        `WITH todo AS (
+           SELECT r.group_id, min(r.priority) AS priority, sum(r.end_id - r.next_id + 1) AS articles
+           FROM ranges r
+           WHERE (r.status = 'planned' OR (r.status = 'leased' AND r.heartbeat_at < now() - make_interval(secs => ?)))
+             AND r.next_id <= r.end_id AND ${todo.sql}
+           GROUP BY r.group_id
+         ), free AS (
+           SELECT g.id, g.name, g.priority AS group_priority, l.snapshot_next, l.snapshot_at, l.heartbeat_at,
+                  todo.priority, todo.articles
+           FROM todo JOIN groups g ON g.id = todo.group_id AND g.active
+           LEFT JOIN group_leases l ON l.group_id = g.id AND l.lane = ?
+           WHERE l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?)
+         ), turns AS (
+           SELECT free.*,
+                  least(2 * row_number() OVER (PARTITION BY priority ORDER BY articles DESC, group_priority, id) - ?,
+                        2 * row_number() OVER (PARTITION BY priority ORDER BY heartbeat_at NULLS FIRST, group_priority, id) - ?) AS turn
+           FROM free
+         )
+         SELECT * FROM turns ORDER BY priority, turn, group_priority, id LIMIT 1`,
+        [ttl, ...todo.bindings, lane, ttl, first, second]
+      )
+      const row = rows[0]
+      if (!row) return null
+      const groupId = Number(row.id)
+      // What the snapshot doesn't cover is read again: the lane's ranges from its article on that the window did
+      // after it (a range done before the snapshot was taken — by the old range leases, say — stays done).
+      if (row.snapshot_next !== null) {
+        await trx.rawQuery(
+          `UPDATE ranges r SET status = 'planned', node_id = NULL, next_id = greatest(r.start_id, ?::bigint)
+           WHERE r.group_id = ? AND r.end_id >= ?::bigint AND ${all.sql}
+             AND (r.status = 'leased' OR (r.status = 'done' AND (r.done_at IS NULL OR r.done_at >= ?)) OR (r.status = 'planned' AND r.next_id > ?::bigint))`,
+          [
+            row.snapshot_next,
+            groupId,
+            row.snapshot_next,
+            ...all.bindings,
+            row.snapshot_at ?? new Date(0),
+            row.snapshot_next,
+          ]
+        )
+      } else {
+        await trx.rawQuery(
+          `UPDATE ranges r SET status = 'planned', node_id = NULL WHERE r.group_id = ? AND r.status = 'leased' AND ${all.sql}`,
+          [groupId, ...all.bindings]
+        )
+      }
+      await trx.rawQuery(
+        `INSERT INTO group_leases (group_id, lane, node_id, leased_at, heartbeat_at, range_id)
+         VALUES (?, ?, ?, now(), now(), NULL)
+         ON CONFLICT (group_id, lane) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
+        [groupId, lane, nodeId]
+      )
+      await trx.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
+      return {
+        group: { id: groupId, name: row.name, priority: Number(row.group_priority) },
+        lane,
+        rules,
+        snapshot: row.snapshot_next !== null ? { next: Number(row.snapshot_next) } : null,
+      }
+    })
+  }
+
+  /** The group's lease row of the lane, when this node holds it. */
+  async held(groupId: number, nodeId: number, lane: Lane = 'history') {
+    const lease = await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', lane)
+      .first()
+    if (!lease || Number(lease.node_id) !== nodeId)
+      throw new LeaseLost(`Group ${groupId} (${lane}) is not held by this node`)
+    return lease
+  }
+
+  /**
+   * The group's next range of the lane in article order (or the one this node has), leased to the node; null when
+   * the lane is caught up.
+   */
+  async next(groupId: number, nodeId: number, lane: Lane = 'history'): Promise<NextRange | null> {
+    await this.held(groupId, nodeId, lane)
+    const rules = await RuleSets.current()
+    const todo = this.laneFilter(lane, true)
+    const all = this.laneFilter(lane, false)
+    const mine = (
+      await db.rawQuery(
+        `SELECT r.id, r.start_id, r.end_id, r.next_id FROM ranges r
+         WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql} ORDER BY r.start_id LIMIT 1`,
+        [groupId, nodeId, ...all.bindings]
+      )
+    ).rows[0]
+    const row =
+      mine ??
+      (
+        await db.rawQuery(
+          `UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
+           WHERE r.id = (
+             SELECT r.id FROM ranges r WHERE r.group_id = ? AND r.status = 'planned' AND r.next_id <= r.end_id AND ${todo.sql}
+             ORDER BY r.start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
+           RETURNING r.id, r.start_id, r.end_id, r.next_id`,
+          [nodeId, rules.version, groupId, ...todo.bindings]
+        )
+      ).rows[0]
+    await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', lane)
+      .update({ range_id: row ? Number(row.id) : null, heartbeat_at: new Date() })
+    if (!row) return null
+    return {
+      range: {
+        id: Number(row.id),
+        start: Number(row.start_id),
+        end: Number(row.end_id),
+        next: Number(row.next_id),
+      },
+    }
+  }
+
+  async heartbeat(groupId: number, nodeId: number, lane: Lane = 'history') {
+    await this.held(groupId, nodeId, lane)
+    await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', lane)
+      .update({ heartbeat_at: new Date() })
+    await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
+  }
+
+  /** The lane a range belongs to, by its priority. */
+  laneOf(priority: number): Lane {
+    return priority <= coordinatorConfig.tail.priority ? 'tail' : 'history'
+  }
+
+  /** A range of the group finished by its holder: counted on the lane's lease (Leases.finish does the range). */
+  async rangeDone(groupId: number, headers: number, lane: Lane = 'history') {
+    await db.rawQuery(
+      `UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ? AND lane = ?`,
+      [headers, groupId, lane]
+    )
+  }
+
+  /** The worker gives the group back (after its snapshot): its range stays at its resume point for the next holder. */
+  async release(groupId: number, nodeId: number, lane: Lane = 'history') {
+    await this.held(groupId, nodeId, lane)
+    const all = this.laneFilter(lane, false)
+    await db.rawQuery(
+      `UPDATE ranges r SET status = 'planned', node_id = NULL
+       WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql}`,
+      [groupId, nodeId, ...all.bindings]
+    )
+    await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', lane)
+      .update({ node_id: null, range_id: null })
+  }
+
+  snapshotPath(groupId: number, lane: Lane = 'history') {
+    return join(this.dir, lane === 'tail' ? `${groupId}-tail.gz` : `${groupId}.gz`)
+  }
+
+  /**
+   * The window's snapshot standing at `next` (the next article to index), streamed to disk (a busy group's is
+   * gigabytes) and written whole before it replaces the old one.
+   */
+  async putSnapshot(
+    groupId: number,
+    nodeId: number,
+    next: number,
+    body: Readable,
+    lane: Lane = 'history'
+  ) {
+    await this.held(groupId, nodeId, lane)
+    await mkdir(this.dir, { recursive: true })
+    const path = this.snapshotPath(groupId, lane)
+    await pipeline(body, createWriteStream(`${path}.part`))
+    await rename(`${path}.part`, path)
+    await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', lane)
+      .update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
+  }
+
+  /** The snapshot's stream and size, or null when there is none. */
+  async getSnapshot(
+    groupId: number,
+    nodeId: number,
+    lane: Lane = 'history'
+  ): Promise<{ stream: Readable; size: number; next: number } | null> {
+    const lease = await this.held(groupId, nodeId, lane)
+    if (lease.snapshot_next === null) return null
+    const path = this.snapshotPath(groupId, lane)
+    try {
+      const { size } = await stat(path)
+      return { stream: createReadStream(path), size, next: Number(lease.snapshot_next) }
+    } catch {
+      return null
+    }
+  }
+}
+
+export default new GroupLeases()

app/services/hash_twins.ts

@@ -0,0 +1,36 @@
+import db from '@adonisjs/lucid/services/db'
+
+/**
+ * Twins by content (LOG.md (130)): a probed file's first 16 KiB and size (file_hashes, what the par2 merge and the
+ * repeated verdict match) in a catalog release that has a name, readable or found inside. The same film posted
+ * under scrambles by other tools gets that name without another fetch. Releases with a junk verdict don't count.
+ */
+/** Twins read per pair: the hottest pairs have 131k. */
+const TWIN_SCAN = 500
+
+export async function twinNames(
+  pairs: { hash16k: string; size: number }[],
+  trx: { rawQuery: typeof db.rawQuery } = db
+): Promise<Map<string, string>> {
+  const out = new Map<string, string>()
+  if (pairs.length === 0) return out
+  // Per pair the first named twin among its lowest TWIN_SCAN release ids. hash16k is typed as the column
+  // (char(32)): as text the join cast the column and scanned all of file_hashes, 4 s a call; and with the index
+  // the hottest pairs (55 with over 500 twins, up to 131k, all readable) made a call walk every twin, 37 s inside
+  // the flush's transaction (LOG.md (138)). One name is all a pair needs.
+  const { rows } = await trx.rawQuery(
+    `SELECT v.hash16k, v.size, t.name
+     FROM jsonb_to_recordset(?::jsonb) AS v(hash16k char(32), size bigint)
+     CROSS JOIN LATERAL (
+       SELECT coalesce(r.inner_name, r.name) AS name
+       FROM (SELECT release_id FROM file_hashes h WHERE h.hash16k = v.hash16k AND h.size = v.size ORDER BY release_id LIMIT ?) h
+       JOIN releases r ON r.id = h.release_id AND (r.readable OR r.inner_name IS NOT NULL)
+       WHERE NOT EXISTS (SELECT 1 FROM release_verdicts x WHERE x.release_id = r.id)
+       ORDER BY r.readable DESC, r.id LIMIT 1) t`,
+    [JSON.stringify(pairs), TWIN_SCAN]
+  )
+  for (const r of rows) out.set(`${r.hash16k}\t${r.size}`, r.name)
+  return out
+}
+
+export const twinKey = (hash16k: string, size: number) => `${hash16k}\t${size}`

app/services/leases.ts

@@ -2,6 +2,8 @@ import db from '@adonisjs/lucid/services/db'
 import coordinatorConfig from '#config/coordinator'
 import RuleSets from '#services/rule_sets'
 import Spool from '#services/spool'
+import Corpus from '#services/corpus'
+import CatalogSpace from '#services/catalog_space'
 import type { Lease } from '#lib/edge/index'
 
 export class LeaseLost extends Error {}
@@ -15,19 +17,36 @@ export class LeaseLost extends Error {}
  * prefetched (LOG.md 2026-10-09). A lease whose worker went quiet for
  * LEASE_TTL_SECONDS is taken over and resumes at its resume point.
  */
+/**
+ * A group's batches wait for each other (the batch store locks the group): more workers than this on one group only
+ * queue. Eight on one picture group's gap ranges (priority -1, in id order) held the node API's batches for minutes
+ * (LOG.md 2026-10-16 (68)).
+ */
+const MAX_LEASES_PER_GROUP = 2
+
 class Leases {
   async lease(nodeId: number): Promise<Lease | null> {
     const rules = await RuleSets.current()
+    // The catalog at its cap on the NVMe: the tail and the stored ranges only (CatalogSpace).
+    const history = (await CatalogSpace.full()) ? 'AND priority < 0' : ''
     const take = (spooled: boolean) =>
       db.rawQuery(
         `UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
          WHERE r.id = (
            SELECT id FROM ranges
-           WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id
+           WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id ${history}
              AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
+             AND (SELECT count(*) FROM ranges l
+                  WHERE l.status = 'leased' AND l.group_id = ranges.group_id AND l.id <> ranges.id
+                    AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
            ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
          RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
-        [nodeId, rules.version, coordinatorConfig.leaseTtlSeconds]
+        [
+          nodeId,
+          rules.version,
+          coordinatorConfig.leaseTtlSeconds,
+          coordinatorConfig.leaseTtlSeconds,
+        ]
       )
     let { rows } = await take(true)
     if (rows.length === 0) ({ rows } = await take(false))
@@ -35,12 +54,38 @@ class Leases {
     if (!row) return null
     await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
     const group = await db.from('groups').where('id', row.group_id).select('name').firstOrFail()
-    return { range: { id: Number(row.id), group: group.name, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) }, rules }
+    return {
+      range: {
+        id: Number(row.id),
+        group: group.name,
+        start: Number(row.start_id),
+        end: Number(row.end_id),
+        next: Number(row.next_id),
+      },
+      rules,
+    }
+  }
+
+  /**
+   * Ranges read to their end whose worker went quiet before finishing them: done. The lease query skips a range past
+   * its end, so one whose finish never came stayed leased for good (a gap range since 2026-10-16 14:03, during the lock
+   * pile-up; LOG.md 2026-10-17 (72)). Its batches are all stored; its spooled chunks go.
+   */
+  async finishAbandoned(): Promise<number> {
+    const { rows } = await db.rawQuery(
+      `UPDATE ranges SET status = 'done', done_at = now()
+       WHERE status = 'leased' AND next_id > end_id AND heartbeat_at < now() - make_interval(secs => ?)
+       RETURNING id`,
+      [coordinatorConfig.leaseTtlSeconds]
+    )
+    for (const { id } of rows) await Spool.remove(Number(id))
+    return rows.length
   }
 
   async leased(rangeId: number, nodeId: number) {
     const range = await db.from('ranges').where('id', rangeId).first()
-    if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
+    if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
+      throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
     return range
   }
 
@@ -53,8 +98,14 @@ class Leases {
   /** The worker reached the range's end: done, and its spooled chunks go. */
   async finish(rangeId: number, nodeId: number) {
     const range = await this.leased(rangeId, nodeId)
-    if (Number(range.next_id) <= Number(range.end_id)) throw new LeaseLost(`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`)
-    await db.from('ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
+    if (Number(range.next_id) <= Number(range.end_id))
+      throw new LeaseLost(
+        `Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`
+      )
+    await db
+      .from('ranges')
+      .where('id', rangeId)
+      .update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
     await Spool.remove(rangeId)
   }
 
@@ -65,9 +116,41 @@ class Leases {
    */
   async corrupt(rangeId: number, nodeId: number, from: number) {
     const range = await this.leased(rangeId, nodeId)
-    const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find((s) => s.from === from)
-    if (span) await Spool.removeChunk(rangeId, span)
-    await db.from('ranges').where('id', rangeId).update({ status: 'planned', node_id: null })
+    const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find(
+      (s) => s.from === from
+    )
+    if (span) {
+      await Spool.removeChunk(rangeId, span)
+      if (range.capture) {
+        const group = await db.from('groups').where('id', range.group_id).select('name').first()
+        await Corpus.removeChunk(
+          { group: group.name, start: Number(range.start_id), end: Number(range.end_id) },
+          span
+        )
+        // Not complete in the corpus any more: the corpus fill fetches the chunk again.
+        await db.from('ranges').where('id', rangeId).update({ stored_at: null })
+      }
+    }
+    const plain: number[] = (range.plain_chunks ?? []).map(Number)
+    // Reported once already: the compressed copy may have been in flight (or the prefetcher's targets not
+    // refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.
+    if (plain.filter((p) => p === from).length >= 2) {
+      // Fetched plain (deflated here) and still not inflating: not the server's compression; set aside.
+      await db
+        .from('ranges')
+        .where('id', rangeId)
+        .update({
+          status: 'failed',
+          node_id: null,
+          error: `chunk ${from} doesn't inflate, also fetched plain`,
+        })
+      return
+    }
+    // The server's compressed reply is cut short (the same bytes every time): fetched plain next time.
+    await db.rawQuery(
+      `UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`,
+      [from, rangeId]
+    )
   }
 
   /** A stopping worker gives its range back; the next one resumes it. */

app/services/nntp_pool.ts

@@ -2,25 +2,89 @@ import env from '#start/env'
 import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import { XoverClient } from '#lib/nntp/index'
+import type { NntpShare } from '#services/nntp_shares'
+
+/** Connections wanted beyond the most in use at once since the last sync (the planner's, a hedge's). */
+const SHARE_MARGIN = 2
 
 /**
  * The coordinator's header connections (REBUILD.md, decision e: workers
  * have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
- * that failed is closed and replaced on the next use.
+ * that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
+ * replaced on the next use. Sharing the account (share()), at most its
+ * share: over it, connections close as they come free. One left free for
+ * NNTP_IDLE_CLOSE_SECONDS is closed on the next sync.
  */
 class NntpPool {
   private idle: XoverClient[] = []
   private busy = new Set<XoverClient>()
   private open = 0
   private waiting: ((client: XoverClient) => void)[] = []
+  private openedAt = new Map<XoverClient, number>()
+  /** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
+  private retireAt = new Map<XoverClient, number>()
+  /** When an idle connection came free. */
+  private idleSince = new Map<XoverClient, number>()
+  private limit = coordinatorConfig.nntpConnections
+  /** Since the last sync: the most connections in use at once, and whether work waited for one (starved()). */
+  private peak = 0
+  private short = false
 
   get size() {
-    return coordinatorConfig.nntpConnections
+    return this.limit
+  }
+
+  /** Work that could use another connection didn't get one (the prefetcher with all its loops running). */
+  starved() {
+    this.short = true
+  }
+
+  /**
+   * Tells the share what the pool could use and has open, and takes the share's grant as its size. After work waited:
+   * twice its size (history going on gets its maximum in a few syncs; a tail round, done in seconds, doesn't take
+   * the prober's connections for one); else the most in use at once since the last sync and a margin.
+   */
+  async share(share: NntpShare) {
+    const want = this.short || this.waiting.length > 0 ? Math.max(this.limit * 2, this.limit + SHARE_MARGIN) : this.peak + SHARE_MARGIN
+    this.short = false
+    this.peak = this.busy.size
+    const granted = await share.sync(want, this.open)
+    const was = this.limit
+    this.limit = granted
+    // Over the share: the idle ones close now, the busy ones once free (release()).
+    while (this.open > this.limit && this.idle.length > 0) this.retire(this.idle.pop()!)
+    this.closeIdle()
+    // Under it with work waiting: connections for it.
+    while (this.waiting.length > 0 && this.open < this.limit) {
+      const next = this.waiting.shift()!
+      this.acquire().then(next, () => this.waiting.unshift(next))
+    }
+    return { was, granted }
   }
 
   async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
-    const client = await this.acquire()
+    return this.run(await this.acquire(), work)
+  }
+
+  /** As use(), if a connection is free or can be opened now; else null, without waiting. */
+  tryUse<T>(work: (client: XoverClient) => Promise<T>): Promise<T> | null {
+    if (this.idle.length === 0 && this.open >= this.size) return null
+    return this.acquire().then((client) => this.run(client, work))
+  }
+
+  /** Connections with a command in flight (the prefetcher's, the planner's). */
+  get busyCount() {
+    return this.busy.size
+  }
+
+  /** How long ago the connection was opened, in ms. */
+  ageOf(client: XoverClient): number {
+    return Date.now() - (this.openedAt.get(client) ?? Date.now())
+  }
+
+  private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
     this.busy.add(client)
+    this.peak = Math.max(this.peak, this.busy.size)
     let broken = false
     try {
       return await work(client)
@@ -33,13 +97,32 @@ class NntpPool {
     }
   }
 
+  /** Closes the connections free for nntpIdleCloseMs, the plain one too. */
+  private closeIdle() {
+    const idleMs = coordinatorConfig.nntpIdleCloseMs
+    if (idleMs <= 0) return
+    const before = Date.now() - idleMs
+    for (const client of this.idle.filter((c) => (this.idleSince.get(c) ?? Infinity) <= before)) {
+      this.idle.splice(this.idle.indexOf(client), 1)
+      this.retire(client)
+    }
+    if (this.plain && this.plainPending === 0 && this.plainUsedAt <= before) {
+      void this.plain.then((c) => c.close()).catch(() => {})
+      this.plain = null
+    }
+  }
+
   private async acquire(): Promise<XoverClient> {
+    // The one used last: the others stay free long enough to close.
     const ready = this.idle.pop()
-    if (ready) return ready
+    if (ready) {
+      this.idleSince.delete(ready)
+      return ready
+    }
     if (this.open < this.size) {
       this.open++
       try {
-        return await XoverClient.connect({
+        const client = await XoverClient.connect({
           host: env.get('NNTP_HOST')!,
           port: env.get('NNTP_PORT', 563),
           user: env.get('NNTP_USER'),
@@ -48,6 +131,10 @@ class NntpPool {
           compress: true,
           timeoutMs: coordinatorConfig.nntpTimeoutMs,
         })
+        this.openedAt.set(client, Date.now())
+        const maxMs = coordinatorConfig.nntpConnectionMaxMs
+        if (maxMs > 0) this.retireAt.set(client, Date.now() + maxMs * (0.75 + Math.random() / 2))
+        return client
       } catch (error) {
         this.open--
         throw error
@@ -57,13 +144,12 @@ class NntpPool {
   }
 
   private release(client: XoverClient, broken: boolean) {
-    if (broken) {
-      this.open--
-      try {
-        client.close()
-      } catch (error: any) {
-        logger.debug(`Closing a failed NNTP connection: ${error.message}`)
-      }
+    if (this.open > this.limit) {
+      this.retire(client)
+      return
+    }
+    if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
+      this.retire(client)
       // Someone waiting gets a fresh connection.
       const next = this.waiting.shift()
       if (next) this.acquire().then(next, () => this.waiting.unshift(next))
@@ -71,13 +157,72 @@ class NntpPool {
     }
     const next = this.waiting.shift()
     if (next) next(client)
-    else this.idle.push(client)
+    else {
+      this.idle.push(client)
+      this.idleSince.set(client, Date.now())
+    }
+  }
+
+  private retire(client: XoverClient) {
+    this.open--
+    this.openedAt.delete(client)
+    this.retireAt.delete(client)
+    this.idleSince.delete(client)
+    try {
+      client.close()
+    } catch (error: any) {
+      logger.debug(`Closing an NNTP connection: ${error.message}`)
+    }
   }
 
   /** Closes every connection, also those in use: their commands fail. */
+  private plain: Promise<XoverClient> | null = null
+  private plainQueue: Promise<unknown> = Promise.resolve()
+  /** Commands queued or running on the plain connection, and when the last one ended. */
+  private plainPending = 0
+  private plainUsedAt = 0
+
+  /**
+   * One connection without XFEATURE COMPRESS GZIP, opened when first needed,
+   * one command at a time: for chunks whose compressed reply the server
+   * cuts short (Leases.corrupt). Its replies are deflated here.
+   */
+  async usePlain<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
+    this.plainPending++
+    const run = this.plainQueue.then(async () => {
+      this.plain ??= XoverClient.connect({
+        host: env.get('NNTP_HOST')!,
+        port: env.get('NNTP_PORT', 563),
+        user: env.get('NNTP_USER'),
+        password: env.get('NNTP_PASSWORD'),
+        secure: env.get('NNTP_SECURE', true),
+        compress: false,
+        timeoutMs: coordinatorConfig.nntpTimeoutMs,
+      })
+      const client = await this.plain
+      try {
+        return await work(client)
+      } catch (error) {
+        this.plain = null
+        client.close()
+        throw error
+      }
+    })
+    this.plainQueue = run.catch(() => {}).finally(() => {
+      this.plainPending--
+      this.plainUsedAt = Date.now()
+    })
+    return run
+  }
+
   async close() {
+    void this.plain?.then((c) => c.close()).catch(() => {})
+    this.plain = null
     for (const client of [...this.idle.splice(0), ...this.busy]) client.close()
     this.busy.clear()
+    this.openedAt.clear()
+    this.retireAt.clear()
+    this.idleSince.clear()
     this.open = 0
   }
 }

app/services/nntp_shares.ts

@@ -0,0 +1,90 @@
+import db from '@adonisjs/lucid/services/db'
+import coordinatorConfig from '#config/coordinator'
+
+/** A share whose process hasn't synced for this long is gone: its connections closed with it. */
+const STALE_SECONDS = 60
+/** Syncs one at a time (pg_advisory_xact_lock). */
+const LOCK = 2_026_101_875
+
+export interface Share {
+  name: string
+  priority: number
+  /** Kept whatever the others want (as far as the total goes, by priority). */
+  min: number
+  /** What it could use now. */
+  want: number
+  /** Connections it has open (or may have: a lane without one yet counts). */
+  open: number
+}
+
+/** What each share should have: its minimum first, then by priority what it wants, up to the total. */
+export function targets(total: number, shares: Share[]): Map<string, number> {
+  const ordered = [...shares].sort((a, b) => b.priority - a.priority || a.name.localeCompare(b.name))
+  const target = new Map<string, number>()
+  let left = total
+  for (const s of ordered) {
+    const t = Math.min(s.min, left)
+    target.set(s.name, t)
+    left -= t
+  }
+  for (const s of ordered) {
+    const more = Math.min(Math.max(s.want - target.get(s.name)!, 0), left)
+    target.set(s.name, target.get(s.name)! + more)
+    left -= more
+  }
+  return target
+}
+
+/**
+ * The connections `self` may have: its target, but no more than the others leave over. Another share holds what it
+ * has open or was granted, whichever is more (it may open up to its grant until its next sync tells it otherwise),
+ * so the account never goes over the total: a share grows only into what the others closed or gave up.
+ */
+export function grant(total: number, self: Share, others: (Share & { granted: number })[]): number {
+  const target = targets(total, [self, ...others]).get(self.name)!
+  const held = others.reduce((sum, o) => sum + Math.max(o.open, o.granted), 0)
+  return Math.max(0, Math.min(target, total - held))
+}
+
+/**
+ * A process's share of the NNTP account's connections (LOG.md 2026-10-18 (75)): the coordinator and the prober had a
+ * fixed split, and with history paused the coordinator's 38 stood idle while the prober's 8 were its limit. Each
+ * syncs every nntpShares.everyMs (what it wants and has open) and gets what it may open; one wanting less closes
+ * connections as they come free, and the other grows as those show up closed.
+ */
+export class NntpShare {
+  granted = 0
+
+  constructor(
+    readonly name: string,
+    private priority: number,
+    readonly min: number,
+    readonly max: number
+  ) {}
+
+  async sync(want: number, open: number): Promise<number> {
+    const self: Share = { name: this.name, priority: this.priority, min: this.min, want: Math.min(Math.max(want, this.min), this.max), open }
+    const total = coordinatorConfig.nntpShares.total
+    this.granted = await db.transaction(async (trx) => {
+      await trx.rawQuery(`SELECT pg_advisory_xact_lock(?)`, [LOCK])
+      const { rows } = await trx.rawQuery(
+        `SELECT name, priority, min, want, open, granted FROM nntp_shares WHERE name <> ? AND heartbeat_at > now() - make_interval(secs => ?)`,
+        [this.name, STALE_SECONDS]
+      )
+      const granted = grant(total, self, rows)
+      await trx.rawQuery(
+        `INSERT INTO nntp_shares (name, priority, min, want, open, granted, heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, now())
+         ON CONFLICT (name) DO UPDATE SET priority = excluded.priority, min = excluded.min, want = excluded.want,
+           open = excluded.open, granted = excluded.granted, heartbeat_at = now()`,
+        [this.name, this.priority, this.min, self.want, open, granted]
+      )
+      return granted
+    })
+    return this.granted
+  }
+
+  /** Stopping: its connections are closed, the others needn't wait for the row to go stale. */
+  async leave() {
+    await db.from('nntp_shares').where('name', this.name).delete()
+  }
+}

app/services/nzb_service.ts

@@ -1,35 +1,65 @@
+import { gunzipSync } from 'node:zlib'
 import db from '@adonisjs/lucid/services/db'
-import { decodeChunk } from '#lib/chunks/index'
+import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
 import { buildNzb } from '#lib/nzb/index'
+import { SOURCES } from '#lib/release/source'
+
+/** Where a known password came from, the most trusted first: the spot's NZB meta, its title, its text. */
+const PASSWORD_SOURCES = ['nzb', 'spot-title', 'spot-text']
 
 /**
- * A release's NZB (REBUILD.md, part 9), from its chunks in the chunk store:
- * every chunk decoded, its files in name order. A file stored twice (the same
- * name and first article, e.g. a range read again) is taken once.
+ * A release's files and NZB (REBUILD.md, part 9), from its chunks in the
+ * chunk store: every chunk decoded, its files in name order. A file stored
+ * twice (the same name and first article, e.g. a range read again) is taken
+ * once. A known password (release_passwords) goes into the NZB's meta, where
+ * SABnzbd and NZBGet take it for unpacking.
  */
 class NzbService {
-  async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
-    const release = await db.from('releases').where('id', releaseId).select('name').first()
-    if (!release) return null
-    const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
-    const files = new Map<string, ReturnType<typeof decodeChunk>[number]>()
+  /** The release's known passwords, the most trusted first. */
+  async passwords(releaseId: number): Promise<{ password: string; source: string }[]> {
+    const rows = await db.from('release_passwords').where('release_id', releaseId).orderBy('created_at').select('password', 'source')
+    const rank = (source: string) => (PASSWORD_SOURCES.indexOf(source) + PASSWORD_SOURCES.length + 1) % (PASSWORD_SOURCES.length + 1)
+    return rows.sort((a, b) => rank(a.source) - rank(b.source)).map((r) => ({ password: r.password, source: r.source }))
+  }
+
+  async files(releaseId: number, client: any = db): Promise<ChunkFile[]> {
+    const chunks = await client.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
+    const files = new Map<string, ChunkFile>()
     for (const chunk of chunks) {
       for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
     }
+    return [...files.values()].sort((a, b) => a.name.localeCompare(b.name))
+  }
+
+  async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
+    const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
+    if (!release) return null
+    const password = (await this.passwords(releaseId))[0]?.password
+    // A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
+    if (Number(release.source) === SOURCES.spot) {
+      const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
+      if (spot) return { name: release.name, xml: withPassword(gunzipSync(spot.nzb).toString('utf8'), password) }
+    }
     const xml = buildNzb(
-      [...files.values()]
-        .sort((a, b) => a.name.localeCompare(b.name))
-        .map((f) => ({
-          name: f.name,
-          poster: f.poster,
-          date: f.postedAt ?? 0,
-          groups: f.groups,
-          segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
-        })),
-      { title: release.name }
+      (await this.files(releaseId)).map((f) => ({
+        name: f.name,
+        poster: f.poster,
+        date: f.postedAt ?? 0,
+        groups: f.groups,
+        segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
+      })),
+      { title: release.name, password }
     )
     return { name: release.name, xml }
   }
 }
 
+/** A read NZB with the password in its meta, when it has none (the spot's title or text named it, not its NZB). */
+export function withPassword(xml: string, password: string | undefined): string {
+  if (!password || /<meta\s+type=["']password["']/i.test(xml)) return xml
+  const meta = `<meta type="password">${password.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')}</meta>`
+  if (/<head\b[^>]*>/i.test(xml)) return xml.replace(/<head\b[^>]*>/i, (head) => `${head}${meta}`)
+  return xml.replace(/<nzb\b[^>]*>/i, (nzb) => `${nzb}<head>${meta}</head>`)
+}
+
 export default new NzbService()

app/services/par2_completeness.ts

@@ -0,0 +1,53 @@
+import NzbService from '#services/nzb_service'
+
+/** A file a par2 set describes (par2_files). */
+export interface SetFile {
+  name: string
+  length: number
+  hash16k: string
+}
+
+const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
+
+/**
+ * Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
+ * whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
+ * or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
+ * described name among its files) is complete when it has as many files besides par2 as the set describes, and as
+ * many bytes (its articles' bytes, a little over the files' with yEnc).
+ */
+export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
+  const names = new Set(files.map((f) => base(f.name)))
+  const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
+  const byName = set.filter((f) => names.has(base(f.name))).length
+  if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
+  if (byName > 0) return false
+  const data = files.filter((f) => !/\.par2$/i.test(f.name))
+  return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
+}
+
+/**
+ * Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
+ * list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
+ * a transaction that moved chunks (a merge) reads them as moved.
+ */
+export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
+  if (releaseIds.length === 0) return 0
+  const { rows: releases } = await client.rawQuery(
+    `SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
+    [releaseIds.map(String)]
+  )
+  let decided = 0
+  for (const { id } of releases) {
+    const { rows: set } = await client.rawQuery(
+      `SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
+      [id]
+    )
+    if (set.length === 0) continue
+    const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
+    const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
+    await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
+    decided++
+  }
+  return decided
+}

app/services/par2_merger.ts

@@ -0,0 +1,498 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { par2ReleaseName } from '#lib/release/index'
+import { looksRandom } from '#lib/release/random'
+import { SidecarFolder, SIDECAR } from '#services/sidecar_folder'
+import { randomPoster } from '#lib/release/identity'
+import NzbService from '#services/nzb_service'
+import { Par2Pruner } from '#services/par2_pruner'
+import { pruneSevenZipTails } from '#services/seven_zip_tails'
+import { decidePar2Completeness } from '#services/par2_completeness'
+
+/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
+const MAX_SPREAD_MS = 48 * 3600 * 1000
+/**
+ * A set held by more releases nearby than this is one posted again and again, not one upload's par2 (every
+ * "kantodoNNN.bin" test upload carries the same set: 64 releases became one of 631 files; LOG.md 2026-10-12 (30)).
+ */
+const MAX_HOLDERS = 4
+/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
+const MAX_FILES_PER_SET_FILE = 3
+const SLACK_FILES = 10
+/**
+ * A par2 file held by more file rows than this (the same first 16 KiB and size) is a common file, not one upload's:
+ * it matches nothing. One such file was in 190,380 uploads, and a set naming it loaded every one of them; the merger
+ * ran out of heap every few minutes (LOG.md 2026-10-20 (85)). The judge calls 20 releases repeated.
+ */
+const COMMON_FILE = 20
+/**
+ * The name pass: a set's own poster's releases in its group posted this close to it are looked at; at most
+ * NAME_CANDIDATES of them (a poster's uploads of a day).
+ */
+const NAME_SPREAD_MS = 12 * 3600 * 1000
+const NAME_CANDIDATES = 200
+/** Files a set's upload carries besides those it describes: sidecars, its par2 files, a cover or preview. */
+const besideSet = (name: string) => SIDECAR.test(name) || /\.(par2|jpe?g|png)$/i.test(name)
+/** Between rounds. */
+const ROUND_MS = 60_000
+/** A window looks back this far past the previous one's end (rows committed after it was read). */
+const OVERLAP_MS = 60_000
+/**
+ * A round goes through what's new in windows of this much (created_at), its place kept after each: a round behind by
+ * two days of the probe backlog (3.5M sets) never ended, and every restart started it over (LOG.md 2026-10-20 (85)).
+ */
+const WINDOW_MS = 60 * 60_000
+/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
+const FOLD_EVERY_MS = 5 * 60_000
+/** Between par2_files prunes (complete and judged sets' file lists). */
+const PRUNE_EVERY_MS = 60 * 60_000
+/** A prune's share of the merger's time: the rest goes on in the next one. */
+const PRUNE_BUDGET_MS = 2 * 60_000
+/** The merger's row in worker_cursors: the end of its last whole window, kept over a restart. */
+const CURSOR = 'merger'
+
+export interface MergeRound {
+  sets: number
+  merges: number
+  releases: number
+  /** The name pass: merges, releases in them. */
+  nameMerges: number
+  nameReleases: number
+  /** The inner-name pass: stems checked, merges, releases in them. */
+  stems: number
+  stemMerges: number
+  stemReleases: number
+  /** The sidecar fold: candidates looked at, sidecar-only ones among them, folded into their upload. */
+  folded?: { looked: number; sidecars: number; folded: number }
+}
+
+/**
+ * A lone inspection's inner stem: its yEnc name up to the first dot, when 16–64 letters and digits
+ * ("bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001"). The same expression as release_inspections_inner_stem_idx.
+ */
+const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
+
+/**
+ * A file name as the name pass compares it: without its folder, case, spaces and punctuation, its UTF-8 read back when
+ * it was read as Latin-1 ("Vermächtnis" posted, "Vermächtnis" in the par2).
+ */
+export function fileKey(name: string): string {
+  let n = name.replace(/^.*[\\/]/, '')
+  if (/[\u00c2-\u00f4][\u0080-\u00bf]/.test(n)) {
+    const read = Buffer.from(n, 'latin1').toString('utf8')
+    if (!read.includes('\ufffd')) n = read
+  }
+  return n.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
+}
+
+/** The releases holding a par2 file f (its first 16 KiB and size), none when it's a common file (COMMON_FILE). */
+const HOLDERS = `SELECT c.release_id FROM (
+    SELECT l.release_id, count(*) OVER () AS holders
+    FROM (SELECT h.release_id FROM file_hashes h WHERE h.hash16k = f.hash16k AND h.size = f.length LIMIT ${COMMON_FILE + 1}) l
+  ) c WHERE c.holders <= ${COMMON_FILE}`
+
+interface Member {
+  id: number
+  group_id: number
+  files: number
+  posted_at: Date | null
+  readable: boolean
+  poster_id?: number | null
+  /** The poster's text (joined from posters) where a query needs it. */
+  poster?: string | null
+  file_total?: number | null
+}
+
+/**
+ * The par2 merge (REBUILD.md, phase 2): releases whose files a par2 set
+ * describes are one upload. Obfuscated uploads post each file under a name of
+ * its own ("[9/20] - "14f5999a7bd4680b0720e524cac0b3b5"", hep: 13,232
+ * one-file releases), so the header pass can't group them; the prober reads
+ * each such file's first article (a `lone` candidate) and stores the MD5 of
+ * its first 16 KiB, and a par2 body's file list holds the same hash per file,
+ * whatever the file was renamed to. Releases in one group, posted within
+ * MAX_SPREAD_MS of a release that holds the set's par2, become the one with
+ * the most files: its chunks, probe candidates, inspections and hashes, the
+ * sums of its counts, and when none of them has a readable name the name the
+ * par2's file names share. Each merge is kept in release_merges.
+ *
+ * A second pass merges by inner name: the yEnc name in a lone file's first
+ * article keeps the upload's random stem when the subject names each file
+ * apart ("[1/7] - "471044d8…"" holds "bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001";
+ * hep, hephaestus, gyges: every stem under one counter total; LOG.md
+ * 2026-10-12 (32)). Releases whose lone inspections share such a stem, in one
+ * group, by one poster, under the same counter total and within MAX_SPREAD_MS,
+ * are one upload. Only random stems: a readable one is grouped from the
+ * headers already, and a repost would share it.
+ */
+export class Par2Merger {
+  private stopped = false
+  private folder = new SidecarFolder((trx, found, key, via) => this.merge(trx, found, key, via))
+  /** Null until read from worker_cursors (1970 the first time: everything). */
+  private since: Date | null = null
+
+  stop() {
+    this.stopped = true
+  }
+
+  async run() {
+    let prunedAt = 0
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        // First, at start too: a prune needn't wait for a long round.
+        if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
+          prunedAt = Date.now()
+          const pruned = await new Par2Pruner().round(PRUNE_BUDGET_MS)
+          if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged, ${pruned.final} final.`)
+        }
+        const round = await this.round()
+        if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
+        if (round.nameMerges > 0) logger.info(`Merged ${round.nameReleases} releases into ${round.nameMerges} by their par2 set's file names.`)
+        if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
+        if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
+      } catch (error: any) {
+        logger.error(`Par2 merge round failed: ${error.message}`)
+      }
+      for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  /**
+   * The sets and inner stems with something new since the last round (a par2 found, a hash of one of their files, a
+   * lone inspection), a window at a time.
+   */
+  async round(): Promise<MergeRound> {
+    // A Date keeps milliseconds, now() microseconds: rounded up, a row made at now() is before the end.
+    const { rows: now } = await db.rawQuery(`SELECT now() + interval '1 millisecond' AS at`)
+    const end = new Date(now[0].at)
+    if (!this.since) {
+      const saved = await db.from('worker_cursors').where('name', CURSOR).first()
+      this.since = saved ? new Date(saved.at) : new Date(0)
+    }
+    // First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
+    // par2 pass (hours when that starts over).
+    const folded = await this.folder.round()
+    let foldedAt = Date.now()
+    const foldAgain = async () => {
+      if (Date.now() - foldedAt < FOLD_EVERY_MS) return
+      const more = await this.folder.round()
+      folded.looked += more.looked
+      folded.sidecars += more.sidecars
+      folded.folded += more.folded
+      foldedAt = Date.now()
+      if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
+    }
+    const round: MergeRound = { sets: 0, merges: 0, releases: 0, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
+    while (this.since < end) {
+      if (this.stopped) return round
+      const from = await this.firstNew(new Date(this.since.getTime() - OVERLAP_MS), end)
+      const until = from ? new Date(Math.min(from.getTime() + WINDOW_MS, end.getTime())) : end
+      if (from) {
+        const { touched, sets } = await this.setsToMerge(from, until)
+        round.sets += touched
+        for (const setId of sets) {
+          if (this.stopped) return round
+          await foldAgain()
+          // One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
+          const merged = await this.mergeSet(setId.trim()).catch((error) => {
+            logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
+            return 0
+          })
+          if (merged > 0) {
+            round.merges++
+            round.releases += merged + 1
+          }
+        }
+        // Every set the window found, by its files' names: readable files have no hashes (the files of one upload under
+        // names that differ: scene tracks apart from their album's par2, "Vermaechtnis" beside "Vermächtnis"; LOG.md
+        // 2026-10-23 (97)).
+        const { rows: found } = await db.rawQuery(`SELECT DISTINCT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?`, [from, until])
+        for (const { set_id: setId } of found) {
+          if (this.stopped) return round
+          const merged = await this.mergeByName(setId.trim()).catch((error) => {
+            logger.error(`Name merge of set ${setId.trim()} failed: ${error.message}`)
+            return 0
+          })
+          if (merged > 0) {
+            round.nameMerges++
+            round.nameReleases += merged + 1
+          }
+        }
+        const { rows: stems } = await db.rawQuery(
+          `SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i
+           WHERE i.kind = 'lone' AND i.created_at >= ? AND i.created_at <= ? AND ${INNER_STEM} IS NOT NULL`,
+          [from, until]
+        )
+        for (const { stem } of stems) {
+          if (this.stopped) return round
+          if (!looksRandom(stem)) continue
+          round.stems++
+          const merged = await this.mergeStem(stem).catch((error) => {
+            logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
+            return 0
+          })
+          if (merged > 0) {
+            round.stemMerges++
+            round.stemReleases += merged + 1
+          }
+        }
+      }
+      // Only after a whole window: a restart halfway goes on from the window before.
+      this.since = until
+      await db.rawQuery(
+        `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
+        [CURSOR, this.since]
+      )
+    }
+    return round
+  }
+
+  /** When the first row a window reads was made, from `from` up to `end`; null when none (a gap, the nightly pause). */
+  private async firstNew(from: Date, end: Date): Promise<Date | null> {
+    const { rows } = await db.rawQuery(
+      `SELECT least(
+         (SELECT min(created_at) FROM par2_sets WHERE created_at >= ?),
+         (SELECT min(created_at) FROM file_hashes WHERE created_at >= ?),
+         (SELECT min(created_at) FROM release_inspections WHERE created_at >= ?)) AS at`,
+      [from, from, from]
+    )
+    const at = rows[0].at ? new Date(rows[0].at) : null
+    return at && at <= end ? at : null
+  }
+
+  /**
+   * The sets a window touched (a par2 found, a hash of one of their files), and those among them a merge could
+   * change: held or matched by two releases or more. Most have every file in one release already (96% of a backlog
+   * hour's 147k): mergeSet needn't look at them.
+   */
+  private async setsToMerge(from: Date, until: Date): Promise<{ touched: number; sets: string[] }> {
+    const { rows } = await db.rawQuery(
+      `WITH touched AS MATERIALIZED (
+         SELECT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?
+         UNION
+         SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size
+         WHERE h.created_at >= ? AND h.created_at <= ?
+       )
+       SELECT (SELECT count(*) FROM touched) AS touched, array(
+         SELECT t.set_id FROM touched t
+         WHERE EXISTS (
+           SELECT 1 FROM (
+             SELECT s.release_id FROM par2_sets s WHERE s.set_id = t.set_id
+             UNION ALL
+             SELECT m.release_id FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) m WHERE f.set_id = t.set_id
+           ) u HAVING count(DISTINCT u.release_id) >= 2)
+       ) AS sets`,
+      [from, until, from, until]
+    )
+    return { touched: Number(rows[0].touched), sets: rows[0].sets }
+  }
+
+  /**
+   * The releases a set's file names find (mergeByName without merging): its newest holder and the holders near it,
+   * and the releases of its poster in its group posted within NAME_SPREAD_MS whose files the set all names, besides
+   * sidecars, par2 and images. Null when there's nothing to merge. Not a holder further off: the same par2 posted
+   * again years later ("Auto Bild Klassik Februar 02-2014": 2014 and 2026) is a repost (decision d).
+   */
+  async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
+    const { rows: all } = await trx.rawQuery(
+      `SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, p.poster
+       FROM par2_sets s JOIN releases r ON r.id = s.release_id LEFT JOIN posters p ON p.id = r.poster_id
+       WHERE s.set_id = ? ORDER BY r.posted_at DESC NULLS LAST, r.id`,
+      [setId]
+    )
+    const anchor: Member | undefined = all[0]
+    // A random poster (one per file) bounds nothing: those uploads are the hash pass's.
+    if (!anchor?.poster || !anchor.posted_at || randomPoster(anchor.poster)) return null
+    const holders = all.filter(
+      (h: Member) => h.group_id === anchor.group_id && h.posted_at && Math.abs(new Date(h.posted_at).getTime() - new Date(anchor.posted_at!).getTime()) <= NAME_SPREAD_MS
+    )
+    const { rows: described } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
+    if (described.length === 0) return null
+    const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
+    const at = new Date(anchor.posted_at).getTime()
+    const { rows: candidates } = await trx.rawQuery(
+      `SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id FROM releases r
+       WHERE r.poster_id = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
+         AND r.files <= ?
+       ORDER BY r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
+      // More files than the set describes, with its sidecars, can't all be named in it.
+      [anchor.poster_id, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
+    )
+    const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
+    let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
+    for (const c of candidates) {
+      const own = await NzbService.files(Number(c.id), trx)
+      const content = own.filter((f) => !besideSet(f.name))
+      if (content.length === 0 || !content.every((f) => names.has(fileKey(f.name)))) continue
+      found.set(String(c.id), c)
+      files += Number(c.files)
+    }
+    if (found.size < 2) return null
+    // No more files than the set describes, with its par2 volumes and sidecars (as the hash pass).
+    if (files > MAX_FILES_PER_SET_FILE * names.size + SLACK_FILES) return null
+    return found
+  }
+
+  /** Merges the releases whose files one set names (nameMatches); how many went into another. */
+  async mergeByName(setId: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const found = await this.nameMatches(trx, setId)
+      return found ? this.merge(trx, found, setId, 'name') : 0
+    })
+  }
+
+  /** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
+  async mergeStem(stem: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows } = await trx.rawQuery(
+        `SELECT DISTINCT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, r.file_total
+         FROM release_inspections i JOIN releases r ON r.id = i.release_id
+         WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
+        [stem]
+      )
+      const sorted: Member[] = rows.sort((a: Member, b: Member) => (a.posted_at ? new Date(a.posted_at).getTime() : 0) - (b.posted_at ? new Date(b.posted_at).getTime() : 0))
+      const anchor = sorted[0]
+      if (!anchor) return 0
+      const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
+      const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
+      const near = (r: Member) =>
+        r.group_id === anchor.group_id &&
+        (r.poster_id ?? null) === (anchor.poster_id ?? null) &&
+        (total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
+        (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
+      const found = new Map<string, Member>()
+      for (const r of sorted.filter(near)) found.set(String(r.id), r)
+      if (found.size < 2) return 0
+      // No more files than the counter says, with room for par2 volumes posted under it.
+      const counted = Math.max(0, ...[...found.values()].map((r) => total(r) ?? 0))
+      const files = [...found.values()].reduce((n, m) => n + Number(m.files), 0)
+      if (counted > 0 && files > counted + SLACK_FILES) return 0
+      return this.merge(trx, found, stem, 'inner')
+    })
+  }
+
+  /** Merges the releases of one set; how many went into another. */
+  async mergeSet(setId: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows: holders } = await trx.rawQuery(
+        `SELECT r.id, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
+         WHERE s.set_id = ? ORDER BY r.id`,
+        [setId]
+      )
+      const anchor: Member | undefined = holders[0]
+      if (!anchor) return 0
+      const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
+      const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
+      const { rows: matches } = await trx.rawQuery(
+        `SELECT f.file_id, r.id, r.group_id, r.files, r.posted_at, r.readable
+         FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) h JOIN releases r ON r.id = h.release_id
+         WHERE f.set_id = ?`,
+        [setId]
+      )
+      // A file of the set that matches more than one release nearby is ambiguous: files that share their first
+      // 16 KiB and size (131 uploads of 1 GiB test files "kantodoNNN.bin" matched one set's single file and became
+      // one release of 1,284 files; LOG.md 2026-10-12 (16)). A plain repost of the same file stays apart too.
+      const byFile = new Map<string, Map<string, Member>>()
+      for (const m of matches.filter(near)) {
+        const releases = byFile.get(m.file_id) ?? new Map<string, Member>()
+        releases.set(String(m.id), m)
+        byFile.set(m.file_id, releases)
+      }
+      const nearHolders = holders.filter(near)
+      if (nearHolders.length > MAX_HOLDERS) return 0
+      const found = new Map<string, Member>()
+      for (const r of nearHolders) found.set(String(r.id), r)
+      for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r)
+      if (found.size < 2) return 0
+      const { rows: described } = await trx.rawQuery(`SELECT count(*) AS n FROM par2_files WHERE set_id = ?`, [setId])
+      const setFiles = Math.max(Number(described[0].n), 1)
+      if ([...found.values()].reduce((n, m) => n + Number(m.files), 0) > MAX_FILES_PER_SET_FILE * setFiles + SLACK_FILES) return 0
+      return this.merge(trx, found, setId, 'par2')
+    })
+  }
+
+  /**
+   * Merges the found releases into the one with the most files (named by the par2 set's files when none
+   * is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
+   */
+  private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar' | 'name'): Promise<number> {
+    {
+      // Locked in id order (the batch store and other rounds take the same rows).
+      const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
+      const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!)
+      if (members.length < 2) return 0
+      const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a))
+      const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id))
+      const name = via === 'par2' && members.every((m) => !m.readable) ? await this.setName(trx, key) : undefined
+
+      await trx.rawQuery(
+        `INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
+         SELECT id, encode(release_key, 'hex'), ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
+        [target.id, key, via, others]
+      )
+      // A candidate of an article the target, or another member of lower id, has already (the same file in two releases)
+      // isn't moved: the name pass met them (every round of 3 sets failed on the unique key; LOG.md 2026-10-23 (99)).
+      await trx.rawQuery(
+        `DELETE FROM probe_candidates p WHERE p.release_id = ANY(?)
+           AND (EXISTS (SELECT 1 FROM probe_candidates t WHERE t.release_id = ? AND t.kind = p.kind AND t.message_id = p.message_id)
+             OR EXISTS (SELECT 1 FROM probe_candidates o WHERE o.release_id = ANY(?) AND o.release_id < p.release_id AND o.kind = p.kind AND o.message_id = p.message_id))`,
+        [others, target.id, others]
+      )
+      for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
+        await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      }
+      // An unsettled member's candidates into a settled target: the settler won't come by again to make them claimable.
+      await trx.rawQuery(
+        `UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
+         WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`,
+        [target.id]
+      )
+      // The members' 7z tails together: those below the highest volume go.
+      await pruneSevenZipTails(trx, [target.id])
+      // A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
+      // round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
+      await trx.rawQuery(
+        `DELETE FROM par2_sets p WHERE p.release_id = ANY(?)
+           AND (EXISTS (SELECT 1 FROM par2_sets t WHERE t.set_id = p.set_id AND t.release_id = ?)
+             OR EXISTS (SELECT 1 FROM par2_sets o WHERE o.set_id = p.set_id AND o.release_id = ANY(?) AND o.release_id < p.release_id))`,
+        [others, target.id, others]
+      )
+      await trx.rawQuery(`UPDATE par2_sets SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      await trx.rawQuery(
+        `UPDATE releases t SET
+           files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
+           first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
+           posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
+           first_seen_at = least(t.first_seen_at, s.first_seen_at),
+           readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
+           file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
+           name = coalesce(nullif(?, ''), t.name)
+         FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
+                 min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
+                 max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at,
+                 bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
+                 max(file_total) AS file_total, max(inner_name) AS inner_name
+               FROM releases WHERE id = ANY(?)) s
+         WHERE t.id = ?`,
+        [name ?? '', others, target.id]
+      )
+      // Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
+      await trx.rawQuery(
+        `UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
+        [target.id]
+      )
+      await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
+      await decidePar2Completeness(trx, [target.id])
+      return others.length
+    }
+  }
+
+  /** The name a set's files share (par2ReleaseName), for an upload none of whose names is readable. */
+  private async setName(trx: any, setId: string): Promise<string | undefined> {
+    const { rows } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
+    const name = par2ReleaseName(rows.map((r: { name: string }) => r.name))
+    // A par2 made after the renaming names the random names again.
+    return name && !looksRandom(name) ? name.slice(0, 255) : undefined
+  }
+}

app/services/par2_pruner.ts

@@ -0,0 +1,113 @@
+import db from '@adonisjs/lucid/services/db'
+import { decidePar2Completeness } from '#services/par2_completeness'
+
+/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
+const SETTLED_MS = 2 * 86400_000
+/** Sets per statement. */
+const BATCH = 20_000
+/** The sweep's row in worker_cursors: the last set_id it looked at (key); past the last set it starts over. */
+const CURSOR = 'par2-prune'
+/** The merger's row in worker_cursors: the end of its last whole window. */
+const MERGER = 'merger'
+/** The merger's windows look back this far (Par2Merger's OVERLAP_MS). */
+const MERGER_OVERLAP_MS = 60_000
+/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
+const JUNK = ['dead', 'fake', 'repeated']
+/** Probe candidates whose results a merge reads: a file's hash (lone), a set's file list (par2). */
+const MERGE_KINDS = ['lone', 'par2']
+
+export interface PruneRound {
+  complete: number
+  judged: number
+  final: number
+  rows: number
+}
+
+/**
+ * Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60), 2026-10-20 (87)). A set's file list
+ * matches file hashes to releases and names an unreadable upload once; it's done when
+ * - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
+ *   set (all of it merged into one);
+ * - judged: every release holding it has a junk verdict (dead, fake, repeated);
+ * - final: SETTLED_MS after its last holder's par2, every holder is final (release_final: a run of done ranges holds
+ *   it with hours to spare, so no more of its upload will be read), the candidates of the ranges around it are probed,
+ *   and the merger has seen every hash of its files. A file still missing then is one no probe will hash: most sets
+ *   describe a readable upload's files, which aren't lone candidates (9 in 10 sets were kept for that, 19 GB).
+ * par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
+ * par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
+ * The sweep goes through the sets with rows by set_id and starts over at the end: a set kept now (a holder at the edge
+ * of a run, its ranges' probes not done yet) is looked at again on the next pass.
+ */
+export class Par2Pruner {
+  /** Sets per batch (BATCH; smaller in the tests). */
+  constructor(private batch = BATCH) {}
+
+  /**
+   * Batches until a pass is done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from
+   * it): run from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md
+   * 2026-10-17 (72)).
+   */
+  async round(budgetMs = Infinity): Promise<PruneRound> {
+    const deadline = Date.now() + budgetMs
+    const round: PruneRound = { complete: 0, judged: 0, final: 0, rows: 0 }
+    const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
+    let after: string = cursors[0]?.key ?? ''
+    // Ranges done since the oldest merge-kind candidate still unprobed: their releases' hashes and file lists may be
+    // still to come. Hashes since the merger's last window: it hasn't matched them yet.
+    const { rows: marks } = await db.rawQuery(
+      `SELECT coalesce((SELECT min(created_at) FROM probe_candidates WHERE status IN ('waiting', 'taken') AND kind = ANY(?)), now()) AS probed,
+              coalesce((SELECT at FROM worker_cursors WHERE name = ?), to_timestamp(0)) - make_interval(secs => ?) AS merged,
+              now() - make_interval(secs => ?) AS settled`,
+      [MERGE_KINDS, MERGER, MERGER_OVERLAP_MS / 1000, SETTLED_MS / 1000]
+    )
+    const { probed, merged, settled } = marks[0]
+    for (;;) {
+      const { rows: sets } = await db.rawQuery(
+        `WITH batch AS MATERIALIZED (SELECT DISTINCT set_id FROM par2_files WHERE set_id > ? ORDER BY set_id LIMIT ?),
+         recent AS MATERIALIZED (SELECT group_id, posted_low, posted_high FROM ranges WHERE status = 'done' AND done_at >= ?),
+         unseen AS MATERIALIZED (SELECT DISTINCT hash16k, size FROM file_hashes WHERE created_at >= ?)
+         SELECT b.set_id, CASE
+           WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
+            AND NOT EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = b.set_id AND NOT EXISTS (
+                  SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
+                  WHERE h.hash16k = f.hash16k AND h.size = f.length)) THEN 'complete'
+           WHEN NOT EXISTS (SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts v ON v.release_id = o.release_id AND v.rule = ANY(?)
+                  WHERE o.set_id = b.set_id AND v.release_id IS NULL) THEN 'judged'
+           WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
+            AND NOT EXISTS (SELECT 1 FROM par2_sets s JOIN releases r ON r.id = s.release_id WHERE s.set_id = b.set_id AND (
+                  r.settled_at IS NULL
+                  OR EXISTS (SELECT 1 FROM recent x WHERE x.group_id = r.group_id
+                    AND x.posted_low <= coalesce(r.last_posted_at, r.posted_at) + interval '6 hours' AND x.posted_high >= r.posted_at - interval '6 hours')))
+            AND NOT EXISTS (SELECT 1 FROM par2_files f JOIN unseen u ON u.hash16k = f.hash16k AND u.size = f.length WHERE f.set_id = b.set_id)
+           THEN 'final' END AS reason
+         FROM batch b ORDER BY b.set_id`,
+        [after, this.batch, probed, merged, settled, JUNK, settled]
+      )
+      const done = (reason: keyof PruneRound) => sets.filter((s: { reason: string }) => s.reason === reason).map((s: { set_id: string }) => s.set_id)
+      for (const reason of ['complete', 'judged', 'final'] as const) {
+        const ids = done(reason)
+        round[reason] += ids.length
+        // The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
+        if (reason !== 'judged' && ids.length > 0) {
+          const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
+          await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
+        }
+        round.rows += await this.drop(ids)
+      }
+      // Past the last set: the next batch starts the next pass.
+      after = sets.length < this.batch ? '' : sets.at(-1).set_id
+      await db.rawQuery(
+        `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
+        [CURSOR, after]
+      )
+      if (after === '' || Date.now() > deadline) break
+    }
+    return round
+  }
+
+  private async drop(setIds: string[]): Promise<number> {
+    if (setIds.length === 0) return 0
+    const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
+    return result.rowCount ?? 0
+  }
+}

app/services/pending_ladder.ts

@@ -0,0 +1,514 @@
+import db from '@adonisjs/lucid/services/db'
+import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
+import coordinatorConfig from '#config/coordinator'
+import { explored } from '#lib/rules/index'
+import { usableName, type Inspection } from '#lib/probe/index'
+import RuleSets from '#services/rule_sets'
+import { twinKey, twinNames } from '#services/hash_twins'
+
+/** pending_releases.status (migration 6400). */
+export const PENDING = {
+  waiting: 0,
+  named: 1,
+  discarded: 2,
+  exhausted: 3,
+  naming: 4,
+  taken: 9,
+} as const
+
+/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
+export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
+
+export interface PendingTarget {
+  kind: string
+  messageId: string
+  file: string
+  /** The target article's bytes (candidates from 2026-11-02 on). */
+  bytes?: number
+}
+
+/** One probe the ladder hands the prober: the next target of a pending release. */
+export interface PendingProbe {
+  /** pending_releases.release_key as hex. */
+  releaseKey: string
+  kind: string
+  messageId: string
+  file: string
+  tool: string
+  groupId: number
+  /** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
+  klass: number
+  poster: string
+  /** Targets probed before this one. */
+  tries: number
+  targets: number
+}
+
+export type PendingOutcome = {
+  probe: PendingProbe
+  status: 'done' | 'missing' | 'error' | 'waiting'
+  inspection?: Inspection
+}
+
+export function byEffort(targets: PendingTarget[]): PendingTarget[] {
+  const rank = (kind: string) => {
+    const i = (EFFORT as readonly string[]).indexOf(kind)
+    return i < 0 ? EFFORT.length : i
+  }
+  return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
+}
+
+/** A class's record (pending_classes), for the claim's order. */
+interface ClassRow {
+  tool: string
+  groupId: number
+  klass: number
+  probed: number
+  named: number
+  waiting: number
+}
+
+/** The record is read again this often. */
+const CLASSES_MS = 60_000
+/** A class found short of a claim's share is left alone this long. */
+const EMPTY_MS = 5 * 60_000
+/**
+ * Rows in one order for a statement that locks many of them (pending_releases by key, pending_classes and
+ * pending_signatures by theirs): the batch store, the record, the claim and the sweep all lock the same way, so two
+ * of them can't deadlock on each other (LOG.md (134)).
+ */
+export function lockOrder<T>(rows: Iterable<T>, key: (row: T) => string): T[] {
+  return [...rows].sort((a, b) => {
+    const ka = key(a)
+    const kb = key(b)
+    return ka < kb ? -1 : ka > kb ? 1 : 0
+  })
+}
+
+/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
+const PRIOR_RATE = 0.15
+const PRIOR_WEIGHT = 10
+/** One class takes at most this share of a claim: the rest explores the classes after it. */
+const CLASS_SHARE = 0.25
+
+/** The expected naming rate of a class: its record shrunk towards the prior. */
+export function classScore(c: { probed: number; named: number }): number {
+  return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
+}
+
+/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
+function claimed(rows: any[]): any[] {
+  const bytes = (r: any) =>
+    r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
+  return [...rows].sort(
+    (a, b) =>
+      bytes(a) - bytes(b) ||
+      new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
+      Number(a.first_article) - Number(b.first_article)
+  )
+}
+
+function toProbe(r: any): PendingProbe {
+  const targets = byEffort(r.candidates as PendingTarget[])
+  const tries = Number(r.tries)
+  const target = targets[Math.min(tries, targets.length - 1)]
+  return {
+    releaseKey: r.release_key,
+    kind: target.kind,
+    messageId: target.messageId,
+    file: target.file,
+    tool: r.tool,
+    groupId: Number(r.group_id),
+    klass: Number(r.klass ?? 0),
+    poster: r.poster,
+    tries,
+    targets: targets.length,
+  }
+}
+
+/**
+ * The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
+ * handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
+ * every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
+ * class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
+ * or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
+ * a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
+ * classes with the best record first (classScore), the cheapest target first within one.
+ */
+class PendingLadder {
+  /** A previous prober's claims back in the queue. */
+  async release(): Promise<number> {
+    const { rowCount } = await db.rawQuery(
+      `UPDATE pending_releases SET status = ? WHERE status = ?`,
+      [PENDING.waiting, PENDING.taken]
+    )
+    return rowCount ?? 0
+  }
+
+  /** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
+  private classes: ClassRow[] = []
+  private classesAt = 0
+  private empty = new Map<string, number>()
+
+  /** The record read again on the next claim (the tests). */
+  forget() {
+    this.classesAt = 0
+    this.empty.clear()
+  }
+
+  /** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
+  private async classRows(): Promise<ClassRow[]> {
+    if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
+    const { rows } = await db.rawQuery(
+      `SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
+    )
+    this.classes = rows
+      .map((r: any) => ({
+        tool: r.tool as string,
+        groupId: Number(r.group_id),
+        klass: Number(r.klass),
+        probed: Number(r.probed),
+        named: Number(r.named),
+        waiting: Number(r.waiting),
+      }))
+      .sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
+    this.classesAt = Date.now()
+    return this.classes
+  }
+
+  /**
+   * The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
+   * at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
+   * and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
+   * classes, status 0 of a class without a row).
+   */
+  async claim(n: number): Promise<PendingProbe[]> {
+    if (n <= 0) return []
+    const out: PendingProbe[] = []
+    const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
+    const now = Date.now()
+    for (const c of await this.classRows()) {
+      if (out.length >= n) break
+      const key = `${c.tool}\t${c.groupId}\t${c.klass}`
+      if ((this.empty.get(key) ?? 0) > now) continue
+      const want = Math.min(perClass, n - out.length)
+      const { rows } = await db.rawQuery(
+        `UPDATE pending_releases p SET status = ? FROM (
+           SELECT release_key FROM pending_releases
+           WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
+           ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
+         WHERE p.release_key = c.release_key
+         RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
+        [PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
+      )
+      if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
+      if (rows.length > 0) {
+        await db.rawQuery(
+          `UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
+          [rows.length, c.tool, c.groupId, c.klass]
+        )
+      }
+      out.push(...claimed(rows).map(toProbe))
+    }
+    if (out.length < n) {
+      const { rows } = await db.rawQuery(
+        `UPDATE pending_releases p SET status = ? FROM (
+           SELECT release_key FROM pending_releases
+           WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
+           ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
+         WHERE p.release_key = c.release_key
+         RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
+        [PENDING.taken, PENDING.waiting, n - out.length]
+      )
+      out.push(...claimed(rows).map(toProbe))
+    }
+    return out
+  }
+
+  /**
+   * What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
+   * end of the ladder (status 3); the finished climbs counted on their signatures.
+   */
+  async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
+    if (outcomes.length === 0) return
+    // A twin by content names what the probe itself didn't (hash_twins.ts).
+    const twins = await twinNames(
+      outcomes
+        .filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
+        .map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
+      trx
+    )
+    const rows = outcomes.map((o) => {
+      const p = o.probe
+      if (o.status === 'waiting') {
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.waiting,
+          tries: p.tries,
+          named: null,
+          inspection: null,
+          final: null,
+        }
+      }
+      const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
+      const twin =
+        o.status === 'done' && o.inspection?.hash16k
+          ? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
+          : undefined
+      const name = probed ?? twin ?? null
+      const tries = p.tries + 1
+      const inspection = {
+        kind: p.kind,
+        status: o.status,
+        result: o.inspection ? { ...o.inspection, par2: undefined } : null,
+      }
+      if (name)
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.naming,
+          tries: 0,
+          named: name,
+          inspection,
+          final: 'named',
+        }
+      if (tries >= p.targets)
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.exhausted,
+          tries,
+          named: null,
+          inspection,
+          final: 'exhausted',
+        }
+      return {
+        release_key: p.releaseKey,
+        status: PENDING.waiting,
+        tries,
+        named: null,
+        inspection,
+        final: null,
+      }
+    })
+    // The rows locked in key order first: the UPDATE's join below takes them in whatever order the plan does.
+    await trx.rawQuery(
+      `SELECT 1 FROM pending_releases WHERE release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
+       ORDER BY release_key FOR UPDATE`,
+      [rows.map((r) => r.release_key).sort()]
+    )
+    await trx.rawQuery(
+      `UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
+         inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
+         probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
+       FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
+       WHERE p.release_key = decode(v.release_key, 'hex')`,
+      [
+        JSON.stringify(
+          lockOrder(
+            rows.map(({ final: _final, ...row }) => row),
+            (r) => r.release_key
+          )
+        ),
+      ]
+    )
+    // The signatures: the poster's own and the tool's and group's together.
+    const counts = new Map<
+      string,
+      { tool: string; group_id: number; poster: string; probed: number; named: number }
+    >()
+    for (const [i, o] of outcomes.entries()) {
+      const final = rows[i].final
+      if (!final) continue
+      for (const poster of [o.probe.poster, '']) {
+        const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
+        const entry = counts.get(key) ?? {
+          tool: o.probe.tool,
+          group_id: o.probe.groupId,
+          poster,
+          probed: 0,
+          named: 0,
+        }
+        entry.probed++
+        if (final === 'named') entry.named++
+        counts.set(key, entry)
+      }
+    }
+    if (counts.size === 0) return
+    const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
+    await trx.rawQuery(
+      `INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
+       SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
+       ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
+      [JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`))]
+    )
+    await trx.rawQuery(
+      `UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float),
+         swept_at = CASE WHEN s.dropped THEN s.swept_at END
+       FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
+       WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
+      [
+        pendingMinProbes,
+        pendingMaxNamed,
+        JSON.stringify(
+          lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`).map(
+            ({ tool, group_id, poster }) => ({ tool, group_id, poster })
+          )
+        ),
+      ]
+    )
+    // The classes: the same record per (tool, group, class), and a probe given back is waiting again.
+    const classes = new Map<
+      string,
+      {
+        tool: string
+        group_id: number
+        klass: number
+        probed: number
+        named: number
+        waiting: number
+      }
+    >()
+    for (const [i, o] of outcomes.entries()) {
+      const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
+      const entry = classes.get(key) ?? {
+        tool: o.probe.tool,
+        group_id: o.probe.groupId,
+        klass: o.probe.klass,
+        probed: 0,
+        named: 0,
+        waiting: 0,
+      }
+      const final = rows[i].final
+      if (final) entry.probed++
+      if (final === 'named') entry.named++
+      if (o.status === 'waiting') entry.waiting++
+      classes.set(key, entry)
+    }
+    await trx.rawQuery(
+      `INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
+       SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
+       ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
+         waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
+      [JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
+    )
+    await trx.rawQuery(
+      `UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float),
+         swept_at = CASE WHEN c.dropped THEN c.swept_at END
+       FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
+       WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
+      [
+        pendingMinProbes,
+        pendingMaxNamed,
+        JSON.stringify(
+          lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`).map(
+            ({ tool, group_id, klass }) => ({ tool, group_id, klass })
+          )
+        ),
+      ]
+    )
+  }
+
+  /**
+   * Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
+   * many. Each dropped class or signature is swept once (swept_at; null again when the record drops it anew),
+   * `perSweep` of them at most, each by its index (the claim's for a class, the signature index for a poster's
+   * rows, the ladder's for a tool's and group's) in rounds of `limit`. The prober runs this beside the lanes, not
+   * in the claim (LOG.md (133)); the edge drops new releases of dropped signatures and classes itself, so one
+   * sweep is all a drop needs.
+   */
+  async discard(limit = 10_000, perSweep = 20): Promise<number> {
+    const { samplePercent } = await RuleSets.current()
+    let discarded = 0
+    const { rows: classes } = await db.rawQuery(
+      `SELECT tool, group_id, klass FROM pending_classes WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, klass LIMIT ?`,
+      [perSweep]
+    )
+    for (const c of classes) {
+      let rows: any[]
+      do {
+        ;({ rows } = await db.rawQuery(
+          `SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
+           WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? LIMIT ?`,
+          [PENDING.waiting, c.tool, c.group_id, c.klass, limit]
+        ))
+        discarded += await this.discardKeys(rows, samplePercent)
+      } while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
+      await db.rawQuery(
+        `UPDATE pending_classes SET swept_at = now() WHERE tool = ? AND group_id = ? AND klass = ?`,
+        [c.tool, c.group_id, c.klass]
+      )
+    }
+    const { rows: signatures } = await db.rawQuery(
+      `SELECT tool, group_id, poster FROM pending_signatures WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, poster LIMIT ?`,
+      [perSweep]
+    )
+    for (const s of signatures) {
+      let rows: any[]
+      do {
+        // The poster's rows by the signature index (on the poster's md5: a poster can be longer than a btree
+        // key), a tool's and group's by the ladder's: never a test on the rows, which was a scan of the whole
+        // table per signature on the HDD (LOG.md (134)).
+        ;({ rows } =
+          s.poster === ''
+            ? await db.rawQuery(
+                `SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
+                 WHERE status = ? AND group_id = ? AND tool = ? LIMIT ?`,
+                [PENDING.waiting, s.group_id, s.tool, limit]
+              )
+            : await db.rawQuery(
+                `SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
+                 WHERE status = ? AND group_id = ? AND tool = ? AND md5(poster) = md5(?::text) AND poster = ? LIMIT ?`,
+                [PENDING.waiting, s.group_id, s.tool, s.poster, s.poster, limit]
+              ))
+        discarded += await this.discardKeys(rows, samplePercent)
+      } while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
+      await db.rawQuery(
+        `UPDATE pending_signatures SET swept_at = now() WHERE tool = ? AND group_id = ? AND poster = ?`,
+        [s.tool, s.group_id, s.poster]
+      )
+    }
+    return discarded
+  }
+
+  /** The rows discarded (status 2), the explored sample excepted, their classes' waiting counts lowered; how many. */
+  private async discardKeys(rows: any[], samplePercent: number): Promise<number> {
+    const keys = rows
+      .map((r: any) => r.release_key as string)
+      .filter((key: string) => !explored(key, samplePercent))
+      .sort()
+    if (keys.length === 0) return 0
+    // Locked in key order (the record's lock order, LOG.md (137)): a plain `key = ANY(subquery)` locks in scan
+    // order and deadlocked against a batch upsert; ORDER BY sorts before FOR UPDATE locks.
+    const { rows: gone } = await db.rawQuery(
+      `UPDATE pending_releases p SET status = ?
+       FROM (SELECT release_key FROM pending_releases
+             WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
+             ORDER BY release_key FOR UPDATE) k
+       WHERE p.release_key = k.release_key
+       RETURNING p.tool, p.group_id, p.klass`,
+      [PENDING.discarded, PENDING.waiting, keys]
+    )
+    const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
+    for (const g of gone) {
+      const key = `${g.tool}\t${g.group_id}\t${g.klass}`
+      const entry = counts.get(key) ?? {
+        tool: g.tool,
+        group_id: Number(g.group_id),
+        klass: Number(g.klass),
+        n: 0,
+      }
+      entry.n++
+      counts.set(key, entry)
+    }
+    if (counts.size > 0) {
+      await db.rawQuery(
+        `UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
+         FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
+         WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
+        [JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
+      )
+    }
+    return gone.length
+  }
+}
+
+export default new PendingLadder()

app/services/pending_namer.ts

@@ -0,0 +1,253 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { promisify } from 'node:util'
+import { unzip } from 'node:zlib'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { RangeIndexer, type ReleaseChunk } from '#lib/edge/index'
+import { identityKey } from '#lib/release/keys'
+import BatchStore from '#services/batch_store'
+import ChunkPartitions from '#services/chunk_partitions'
+import Corpus from '#services/corpus'
+import RuleSets from '#services/rule_sets'
+import { PENDING } from '#services/pending_ladder'
+
+const inflate = promisify(unzip)
+
+interface NamedRow {
+  release_key: Buffer
+  identity_key: Buffer
+  group_id: number
+  named: string
+  first_article: number
+  last_article: number
+  file_total: number | null
+  tries: number
+}
+
+/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
+const MAX_MOVES = 3
+
+/**
+ * The widest article span the namer reads again (a busy group posts 50M articles a day, a large upload among
+ * them spans a million or two): one wider is a name the edge saw again much later ("v-lgpc.3ds"), and its span
+ * of segments would not fit the indexer's memory (LOG.md (133)).
+ */
+const MAX_SPAN = 3_000_000
+
+/**
+ * The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
+ * read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
+ * connection of its own, opened for it and closed after), through the TypeScript indexer, and is written as a
+ * release with the name the probe found (source weak-stem, the name as inner name), its chunk and its probe
+ * candidates, which the prober answers from probe_cache. Runs inside the prober.
+ */
+export class PendingNamer {
+  private client: XoverClient | null = null
+  private usedAt = 0
+
+  constructor(private corpusDir: string | undefined = coordinatorConfig.corpus.dir) {}
+
+  async run(stopped: () => boolean) {
+    while (!stopped()) {
+      let rows: NamedRow[] = []
+      try {
+        rows = await this.take(20)
+        for (const row of rows) {
+          if (stopped()) break
+          try {
+            await this.move(row)
+          } catch (error: any) {
+            logger.warn(
+              `Naming ${row.release_key.toString('hex')} "${row.named}" failed: ${error.message}`
+            )
+            await db.rawQuery(
+              `UPDATE pending_releases SET tries = tries + 1, probed_at = now() WHERE release_key = ?`,
+              [row.release_key]
+            )
+          }
+        }
+      } catch (error: any) {
+        logger.error(`Naming: ${error.message}`)
+      }
+      if (this.client && Date.now() - this.usedAt > coordinatorConfig.nntpIdleCloseMs) {
+        this.client.close()
+        this.client = null
+      }
+      if (rows.length === 0)
+        for (let waited = 0; waited < 30_000 && !stopped(); waited += 1000) await sleep(1000)
+    }
+    this.client?.close()
+    this.client = null
+  }
+
+  /** Named releases awaiting their move, the longest waiting first; those failed MAX_MOVES times stay as they are. */
+  async take(n: number): Promise<NamedRow[]> {
+    const { rows } = await db.rawQuery(
+      `SELECT release_key, identity_key, group_id, named, first_article, last_article, file_total, tries
+       FROM pending_releases WHERE status = ? AND tries < ? ORDER BY probed_at LIMIT ?`,
+      [PENDING.naming, MAX_MOVES, n]
+    )
+    return rows.map((r: any) => ({
+      ...r,
+      group_id: Number(r.group_id),
+      first_article: Number(r.first_article),
+      last_article: Number(r.last_article),
+      tries: Number(r.tries),
+    }))
+  }
+
+  /** One named release into the catalog. */
+  async move(row: NamedRow): Promise<void> {
+    if (row.last_article - row.first_article > MAX_SPAN)
+      throw new Error(
+        `its articles ${row.first_article}–${row.last_article} span more than ${MAX_SPAN}`
+      )
+    const group = await db.from('groups').where('id', row.group_id).firstOrFail()
+    const range = await db
+      .from('ranges')
+      .where('group_id', row.group_id)
+      .where('start_id', '<=', row.first_article)
+      .where('end_id', '>=', row.first_article)
+      .orderBy('id')
+      .first()
+    if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
+    const rules = await RuleSets.current()
+    const ix = new RangeIndexer({
+      group: group.name,
+      start: row.first_article,
+      end: row.last_article,
+      rules: { ...rules, samplePercent: 0 },
+      // No file given up before the end: the span is read chunk by chunk (one text of it topped V8's string).
+      idleChunks: Number.MAX_SAFE_INTEGER,
+      edgeArticles: 0,
+      // The edge wrote it partial when its file is short of parts on the server: so must the re-read.
+      partialFiles: true,
+    })
+    for await (const piece of this.overviewPieces(group.name, row))
+      ix.addText(piece.text, piece.next)
+    ix.finish()
+    const batch = ix.take()
+    const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
+    if (!found)
+      throw new Error(
+        `its articles ${row.first_article}–${row.last_article} of ${group.name} no longer show it`
+      )
+    // The catalog row under the pending key (the identity, the group, the first article the edge saw).
+    const chunk: ReleaseChunk = {
+      ...found,
+      name: row.named,
+      source: 'weak-stem',
+      readable: true,
+      finished: true,
+      edge: false,
+      firstArticle: row.first_article,
+      fileTotal: row.file_total ?? found.fileTotal,
+      tool: undefined,
+    }
+    const candidates = batch.candidates
+      .filter((c) => c.releaseKey === found.key)
+      .map((c) => ({ ...c, upload: found.upload }))
+    // The chunk's month partition, as the batch store makes sure of before its writes.
+    await ChunkPartitions.ensure()
+    await db.transaction(async (trx) => {
+      const blockId = await BatchStore.blockOf(trx, range)
+      await BatchStore.writeReleases(
+        trx,
+        { groupId: row.group_id, blockId },
+        row.first_article,
+        [chunk],
+        candidates
+      )
+      await trx.rawQuery(
+        `UPDATE releases SET inner_name = ?, inspected_at = now() WHERE release_key = ? AND inner_name IS NULL`,
+        [row.named.slice(0, 255), row.release_key]
+      )
+      await trx.rawQuery(
+        `UPDATE pending_releases SET status = ?, probed_at = now() WHERE release_key = ?`,
+        [PENDING.named, row.release_key]
+      )
+    })
+  }
+
+  /**
+   * The overview lines of the release's article span, a chunk of articles at a time with the first article after
+   * it: the stored corpus chunks when its ranges are stored and contiguous, else XOVER in windows of a chunk.
+   */
+  private async *overviewPieces(
+    groupName: string,
+    row: NamedRow
+  ): AsyncGenerator<{ text: string; next: number }> {
+    const size = coordinatorConfig.chunkArticles
+    const stored = await this.storedChunks(groupName, row)
+    if (stored) {
+      for (const { raw, next } of stored)
+        yield { text: (await inflate(raw)).toString('latin1'), next }
+      return
+    }
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: true,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    this.usedAt = Date.now()
+    try {
+      await this.client.group(groupName)
+      for (let from = row.first_article; from <= row.last_article; from += size) {
+        const to = Math.min(from + size - 1, row.last_article)
+        // One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
+        const raw = await this.client.xoverRaw(from, to)
+        yield { text: (await inflate(raw)).toString('latin1'), next: to + 1 }
+      }
+    } catch (error) {
+      this.client.close()
+      this.client = null
+      throw error
+    }
+  }
+
+  /** The span's stored corpus chunks (compressed, with the first article after each), or null when any is missing. */
+  private async storedChunks(
+    groupName: string,
+    row: NamedRow
+  ): Promise<{ raw: Buffer; next: number }[] | null> {
+    if (!this.corpusDir) return null
+    const stored = await db
+      .from('ranges')
+      .where('group_id', row.group_id)
+      .whereNotNull('stored_at')
+      .where('start_id', '<=', row.last_article)
+      .where('end_id', '>=', row.first_article)
+      .orderBy('start_id')
+    if (
+      stored.length === 0 ||
+      Number(stored[0].start_id) > row.first_article ||
+      Number(stored[stored.length - 1].end_id) < row.last_article
+    )
+      return null
+    const size = coordinatorConfig.chunkArticles
+    const chunks: { raw: Buffer; next: number }[] = []
+    for (const r of stored) {
+      const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
+      for (
+        let from =
+          range.start +
+          Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
+        from <= Math.min(row.last_article, range.end);
+        from += size
+      ) {
+        const to = Math.min(from + size - 1, range.end)
+        const raw = await Corpus.read(range, { from, to }, this.corpusDir)
+        if (!raw) return null
+        chunks.push({ raw, next: to + 1 })
+      }
+    }
+    return chunks
+  }
+}

app/services/planner.ts

@@ -35,27 +35,16 @@ class Planner {
 
   async planGroup(group: { id: number; name: string }): Promise<number> {
     const info = await NntpPool.use((client) => client.group(group.name))
-    const size = coordinatorConfig.rangeArticles
     const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
-    const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
-    const span = Math.max(1, info.high - info.low)
-    const rows = []
-    for (let start = from; start <= info.high; start += size) {
-      rows.push({
-        group_id: group.id,
-        start_id: start,
-        end_id: Math.min(start + size - 1, info.high),
-        next_id: start,
-        priority: Math.floor((1000 * (start - info.low)) / span),
-      })
-    }
+    const rows = historyRows(group.id, info, Number(last?.end ?? 0), coordinatorConfig.rangeArticles)
     for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
     await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
     return rows.length
   }
 
+  /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */
   async planAll(): Promise<{ groups: number; ranges: number }> {
-    const groups = await db.from('groups').where('active', true).orderBy('priority').select('id', 'name')
+    const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name')
     let ranges = 0
     for (const group of groups) {
       // One group failing (e.g. gone from the server) doesn't stop the others.
@@ -67,6 +56,236 @@ class Planner {
     }
     return { groups: groups.length, ranges }
   }
+
+  /**
+   * The live tail: ranges for the articles after each planned group's last
+   * range, at the tail's priority (before any history). The newest article
+   * numbers come from LIST ACTIVE, one command per name prefix (GROUP per
+   * group took minutes for 1000 groups).
+   */
+  async planTail(now = new Date()): Promise<TailPlanned> {
+    const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
+    if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 }
+    const { rows: ends } = await db.rawQuery(
+      `SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id,
+              (status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open
+       FROM ranges ORDER BY group_id, start_id DESC`,
+      [coordinatorConfig.tail.priority]
+    )
+    const last = new Map<number, { id: number; start: number; end: number; open: boolean }>(
+      ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }])
+    )
+    const highs = new Map<string, number>()
+    await NntpPool.use(async (client) => {
+      for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
+        for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
+      }
+    })
+    const plan = tailRanges(
+      groups.map((g) => {
+        const l = last.get(Number(g.id))
+        return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null }
+      }),
+      now,
+      { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
+    )
+    return this.applyTail(plan, now)
+  }
+
+  /** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */
+  async applyTail(plan: TailPlan, now = new Date()): Promise<TailPlanned> {
+    const rows = [...plan.rows]
+    let extended = 0
+    if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 }
+    await db.transaction(async (trx) => {
+      for (const e of plan.extended) {
+        const { rows: done } = await trx.rawQuery(
+          `UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`,
+          [e.end_id, e.id, e.from - 1]
+        )
+        if (done.length > 0) extended++
+        else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority })
+      }
+      if (rows.length > 0) await trx.table('ranges').multiInsert(rows)
+      const highs = new Map<number, number>()
+      for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id))
+      for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now })
+    })
+    return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended }
+  }
+
+  /**
+   * Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most
+   * RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail
+   * (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go.
+   */
+  async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> {
+    const { rows: candidates } = await db.rawQuery(
+      `SELECT id, group_id, start_id, end_id, priority FROM ranges
+       WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL
+       ORDER BY group_id, start_id`
+    )
+    const merges = tailMerges(
+      candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })),
+      coordinatorConfig.rangeArticles
+    )
+    const dropped = merges.reduce((n, m) => n + m.drop.length, 0)
+    if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped }
+    await db.transaction(async (trx) => {
+      // The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile.
+      await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
+      const { rows: gone } = await trx.rawQuery(
+        `DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`,
+        [merges.flatMap((m) => m.drop)]
+      )
+      if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`)
+      const { rows: grown } = await trx.rawQuery(
+        `UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v
+         WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`,
+        [merges.map((m) => m.id), merges.map((m) => m.end_id)]
+      )
+      if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`)
+    })
+    return { candidates: candidates.length, merged: merges.length, dropped }
+  }
+}
+
+export interface TailPlanned {
+  groups: number
+  /** Rows added. */
+  ranges: number
+  /** Open ranges grown. */
+  extended: number
+}
+
+/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
+export function wildmatOf(name: string): string {
+  const parts = name.split('.')
+  return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name
+}
+
+export interface TailGroup {
+  id: number
+  /** End of its last range (null: none, then it has no tail yet). */
+  plannedTo: number | null
+  /** When its last range was planned. */
+  plannedAt: Date
+  /** The server's newest article (null: not listed). */
+  high: number | null
+  /**
+   * Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that
+   * nothing spooled, took or tried (its start: the cut at rangeArticles counts from there).
+   */
+  open?: { id: number; start: number } | null
+}
+
+export interface TailRow {
+  group_id: number
+  start_id: number
+  end_id: number
+  next_id: number
+  priority: number
+}
+
+export interface TailPlan {
+  /** New ranges. */
+  rows: TailRow[]
+  /** Open ranges grown to end_id (from: the first new article, the old end + 1). */
+  extended: { id: number; group_id: number; from: number; end_id: number }[]
+}
+
+/**
+ * The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or
+ * the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a
+ * tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`.
+ */
+export function tailRanges(
+  groups: TailGroup[],
+  now: Date,
+  options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
+): TailPlan {
+  const rows: TailRow[] = []
+  const extended: TailPlan['extended'] = []
+  for (const g of groups) {
+    if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
+    const waiting = g.high - g.plannedTo
+    const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
+    if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
+    let start = g.plannedTo + 1
+    if (g.open && g.open.start + options.rangeArticles - 1 >= start) {
+      const end = Math.min(g.open.start + options.rangeArticles - 1, g.high)
+      extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end })
+      start = end + 1
+    }
+    for (; start <= g.high; start += options.rangeArticles) {
+      rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
+    }
+  }
+  return { rows, extended }
+}
+
+export interface TailMerge {
+  /** The run's first range, grown to end_id. */
+  id: number
+  end_id: number
+  /** The ranges it absorbed. */
+  drop: number[]
+}
+
+/**
+ * Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most
+ * `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything.
+ */
+export function tailMerges(
+  ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[],
+  rangeArticles: number
+): TailMerge[] {
+  const merges: TailMerge[] = []
+  let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null
+  const flush = () => {
+    if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop })
+  }
+  for (const r of ranges) {
+    if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) {
+      cur.end_id = r.end_id
+      cur.drop.push(r.id)
+      continue
+    }
+    flush()
+    cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] }
+  }
+  flush()
+  return merges
+}
+
+/**
+ * The rows a group's history is cut into, from `lastEnd + 1` (0: the group has no ranges yet) up to the high
+ * watermark: RANGE_ARTICLES slices at the position priority. A group planned for the first time also gets one
+ * "ancient" range, articles 1 … low − 1: what had expired on the server before it was planned. Status
+ * 'ancient', never leased or fetched; it keeps the tiling complete from article 1 (migration 6700 added it for
+ * the groups planned before 2026-10-28).
+ */
+export function historyRows(
+  groupId: number,
+  info: { low: number; high: number },
+  lastEnd: number,
+  size: number
+): Record<string, unknown>[] {
+  const rows: Record<string, unknown>[] = []
+  if (lastEnd === 0 && info.low > 1)
+    rows.push({ group_id: groupId, start_id: 1, end_id: info.low - 1, next_id: 1, priority: 0, status: 'ancient' })
+  const from = Math.max(info.low, lastEnd + 1)
+  const span = Math.max(1, info.high - info.low)
+  for (let start = from; start <= info.high; start += size) {
+    rows.push({
+      group_id: groupId,
+      start_id: start,
+      end_id: Math.min(start + size - 1, info.high),
+      next_id: start,
+      priority: Math.floor((1000 * (start - info.low)) / span),
+    })
+  }
+  return rows
 }
 
 export default new Planner()

app/services/prefetcher.ts

@@ -3,16 +3,35 @@ import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import NntpPool from '#services/nntp_pool'
 import Spool, { type ChunkSpan } from '#services/spool'
+import Corpus, { type CorpusFilter } from '#services/corpus'
+import RuleSets from '#services/rule_sets'
+import CatalogSpace from '#services/catalog_space'
+import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
 
 interface Target {
   id: number
   group: string
+  /** Chunk starts to fetch without compression (ranges.plain_chunks). */
+  plain: Set<number>
   /** Chunks not spooled yet, in order. */
   missing: ChunkSpan[]
   /** All of the range's chunks, from its start. */
   total: number
+  start: number
+  end: number
+  /** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
+  capture: boolean
+  /** Fetched into the corpus only, not the spool: a capture range no worker leases soon (corpus.ahead). */
+  corpusOnly?: boolean
 }
 
+/** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
+const STALL_SAMPLE_MS = 15_000
+const STALL_SAMPLES = 60_000 / STALL_SAMPLE_MS
+
+/** Connection ages the report counts slow first requests by, in minutes. */
+const AGE_BUCKETS = [2, 10, Infinity]
+
 /** How often the targets are read again from the database. */
 const REFRESH_MS = 2000
 /** How often the spool's size is measured (a stat per file). */
@@ -21,11 +40,13 @@ const SIZE_MS = 30_000
 /**
  * Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
  * Every connection takes the next missing chunk: the leased ranges' first
- * (a worker waits for them; all connections work on the same range, in
- * order), then the next planned ranges, about SPOOL_AHEAD_PER_LEASE per
+ * (a worker waits for them), then the next planned ranges, with at most
+ * PREFETCH_PER_RANGE connections on a range and PREFETCH_PER_GROUP on a
+ * group (Eweka answers faster spread out), about SPOOL_AHEAD_PER_LEASE per
  * active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
  * connection fills ~26 k overview lines/s, so a range on one connection
- * kept its worker waiting (LOG.md 2026-10-09).
+ * kept its worker waiting (LOG.md 2026-10-09). A chunk that takes long is
+ * asked again on a spare connection (fetchHedged).
  */
 export class Prefetcher {
   /**
@@ -37,31 +58,93 @@ export class Prefetcher {
 
   private targets: Target[] = []
   private claimed = new Set<string>()
+  /** Chunks being fetched per range and per group (spool.perRange, spool.perGroup). */
+  private perRange = new Map<number, number>()
+  private perGroup = new Map<string, number>()
   private running = 0
   private refreshedAt = 0
   private stopped = false
   private spoolBytes = 0
   private sizedAt = 0
   /** Fetched since the last report: chunks, articles, ms spent fetching. */
-  private stats = { chunks: 0, articles: 0, fetchMs: 0, since: Date.now() }
+  private stats = this.freshStats()
+  /** Loops in a fetch or a spool write right now. */
+  private fetchingNow = 0
+  private writingNow = 0
+  private sampler?: NodeJS.Timeout
+  /** Requests in flight: what, since when, on which connection (the report names the oldest). */
+  private inFlight = new Set<{
+    rangeId: number
+    from: number
+    to: number
+    since: number
+    client: XoverClient
+  }>()
+  /** Capture ranges found complete in the corpus (not looked at again). */
+  private stored = new Set<number>()
+
+  private freshStats() {
+    return {
+      chunks: 0,
+      /** Chunks read from the corpus instead of fetched. */
+      stored: 0,
+      /** Chunks fetched into the corpus only (corpus.ahead). */
+      captured: 0,
+      articles: 0,
+      fetchMs: 0,
+      hedged: 0,
+      hedgeWon: 0,
+      /** First requests per connection-age bucket, and how many of them were slow. */
+      byAge: AGE_BUCKETS.map(() => ({ requests: 0, slow: 0 })),
+      /** Spool writes and the range's update, ms. */
+      writeMs: 0,
+      /** Per answered request, ms: GROUP, XOVER until its status line, the transfer; transfers with a silence over 5 s. */
+      requests: 0,
+      groupMs: 0,
+      replyMs: 0,
+      bodyMs: 0,
+      silent: 0,
+      /** Once a second: loops fetching, writing, not running; pool connections with a command in flight. */
+      samples: 0,
+      fetching: 0,
+      writing: 0,
+      idle: 0,
+      inFlight: 0,
+      inFlightMin: Infinity,
+      /** Requests given up as stalled (nntpStallBytesPerMinute). */
+      deadlines: 0,
+      ranges: 0,
+      groups: 0,
+      since: Date.now(),
+    }
+  }
+
+  /** The prefetcher's loops: the pool less the connections kept for hedging. */
+  private get loops() {
+    return Math.max(1, NntpPool.size - coordinatorConfig.nntpHedgeConnections)
+  }
 
   /** One round: refreshes the targets now and then, and puts idle connections to work. */
   async tick() {
+    this.sampler ??= setInterval(() => this.sample(), 1000).unref()
     if (Date.now() - this.refreshedAt >= REFRESH_MS) {
       this.refreshedAt = Date.now()
       this.targets = await this.readTargets()
     }
     this.report()
-    while (!this.stopped && this.running < NntpPool.size && this.peek()) {
+    while (!this.stopped && this.running < this.loops && this.peek()) {
       this.running++
       this.work()
         .catch((error) => logger.warn(`Prefetching: ${error.message}`))
         .finally(() => this.running--)
     }
+    // Work left with every loop running: the pool's share could be bigger.
+    if (!this.stopped && this.running >= this.loops && this.peek()) NntpPool.starved()
   }
 
   stop() {
     this.stopped = true
+    clearInterval(this.sampler)
   }
 
   get active() {
@@ -71,58 +154,248 @@ export class Prefetcher {
   /** One connection's loop: chunks until there is nothing left to fetch. */
   private async work() {
     while (!this.stopped) {
+      // The pool's share got smaller: fewer loops (waiting on the pool, they'd count as work short of a connection).
+      // Two leaving at once is one too many for a tick (TICK_MS), whose loop comes back.
+      if (this.running > this.loops) return
       const next = this.take()
       if (!next) return
       const { target, span } = next
       try {
         const fetchedAt = Date.now()
-        const data = await NntpPool.use(async (client) => {
+        this.fetchingNow++
+        const fetch = async (client: Parameters<Parameters<typeof NntpPool.use>[0]>[0]) => {
           await client.group(target.group)
           return client.xoverRaw(span.from, span.to)
-        })
-        this.stats.chunks++
-        this.stats.articles += span.to - span.from + 1
-        this.stats.fetchMs += Date.now() - fetchedAt
-        const existed = await Spool.has(target.id, span)
-        await Spool.write(target.id, span, data)
-        if (existed) continue
-        this.spoolBytes += data.length
-        const { rows } = await db.rawQuery(
-          `UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
-          [target.id]
-        )
-        if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
+        }
+        const corpusRange = { group: target.group, start: target.start, end: target.end }
+        // A range in the corpus is read from disk (a reset re-indexes it without the news server).
+        let data =
+          target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
+        const stored = data !== null
+        try {
+          if (!data)
+            data = target.plain.has(span.from)
+              ? await NntpPool.usePlain(fetch)
+              : await this.fetchHedged(target.id, target.group, span)
+        } finally {
+          this.fetchingNow--
+        }
+        if (stored) {
+          this.stats.stored++
+        } else {
+          this.stats.chunks++
+          this.stats.articles += span.to - span.from + 1
+          this.stats.fetchMs += Date.now() - fetchedAt
+          if (target.capture)
+            await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
+        }
+        if (target.corpusOnly) {
+          this.stats.captured++
+          continue
+        }
+        const writtenAt = Date.now()
+        this.writingNow++
+        try {
+          const existed = await Spool.has(target.id, span)
+          await Spool.write(target.id, span, data)
+          if (existed) continue
+          this.spoolBytes += data.length
+          const { rows } = await db.rawQuery(
+            `UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
+            [target.id]
+          )
+          if (Number(rows[0]?.spooled_chunks) >= target.total)
+            await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
+        } finally {
+          this.writingNow--
+          this.stats.writeMs += Date.now() - writtenAt
+        }
       } finally {
         this.claimed.delete(`${target.id}:${span.from}`)
+        this.count(target, -1)
       }
     }
   }
 
+  /**
+   * A chunk over a pooled connection. Not in after nntpHedgeAfterMs, it's
+   * asked again on a free connection (if there is one): Eweka stalls single
+   * requests, and the same chunk on another connection mostly comes in
+   * seconds (LOG.md 2026-10-09 (15)). The first answer wins; the other
+   * request can't be cancelled, so its connection is closed (the pool opens
+   * a fresh one).
+   */
+  private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
+    let age: number | undefined
+    const request = (
+      start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
+      first: boolean
+    ): Request<Buffer> | null => {
+      let client: XoverClient | undefined
+      let cancelled = false
+      const done = start(async (c) => {
+        // Given up before it got a connection: the connection goes back unused.
+        if (cancelled) return Buffer.alloc(0)
+        client = c
+        if (first) age = NntpPool.ageOf(c)
+        const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c }
+        this.inFlight.add(entry)
+        // Bytes in, sampled every STALL_SAMPLE_MS: less than nntpStallBytesPerMinute over the last minute is a stall.
+        let late = false
+        const samples: number[] = []
+        const watch = setInterval(() => {
+          samples.push(c.buffered)
+          if (samples.length <= STALL_SAMPLES) return
+          const minuteAgo = samples.shift()!
+          if (c.buffered - minuteAgo < coordinatorConfig.nntpStallBytesPerMinute) {
+            late = true
+            this.stats.deadlines++
+            c.close()
+          }
+        }, STALL_SAMPLE_MS)
+        let data: Buffer
+        let groupMs = 0
+        try {
+          const groupAt = performance.now()
+          await c.group(group)
+          groupMs = performance.now() - groupAt
+          data = await c.xoverRaw(span.from, span.to)
+        } catch (error) {
+          if (!late) throw error
+          throw new Error(
+            `Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
+          )
+        } finally {
+          clearInterval(watch)
+          this.inFlight.delete(entry)
+        }
+        if (!cancelled) {
+          const t = c.lastTiming
+          this.stats.requests++
+          this.stats.groupMs += groupMs
+          this.stats.replyMs += t.replyMs
+          this.stats.bodyMs += t.bodyMs
+          if (t.longestGapMs > 5000) this.stats.silent++
+        }
+        return data
+      })
+      if (!done) return null
+      done.catch(() => {})
+      return {
+        done,
+        // The request can't be called off on the wire: its connection is closed, and the pool opens a fresh one.
+        cancel: () => {
+          cancelled = true
+          client?.close()
+        },
+      }
+    }
+    const result = await hedge(
+      () => request((work) => NntpPool.use(work), true)!,
+      () => request((work) => NntpPool.tryUse(work), false),
+      coordinatorConfig.nntpHedgeAfterMs
+    )
+    if (age !== undefined) {
+      const bucket = this.stats.byAge[AGE_BUCKETS.findIndex((minutes) => age! < minutes * 60_000)]
+      bucket.requests++
+      if (result.late) bucket.slow++
+    }
+    if (result.hedged) this.stats.hedged++
+    if (result.secondWon) this.stats.hedgeWon++
+    return result.value
+  }
+
+  /** The longest-running request: range, chunk, how long, how much came in. */
+  private oldest() {
+    let oldest:
+      { rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
+    for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
+    if (!oldest) return 'none in flight'
+    return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
+  }
+
+  private sample() {
+    const s = this.stats
+    s.samples++
+    s.fetching += this.fetchingNow
+    s.writing += this.writingNow
+    s.idle += Math.max(0, this.loops - this.fetchingNow - this.writingNow)
+    s.inFlight += NntpPool.busyCount
+    s.inFlightMin = Math.min(s.inFlightMin, NntpPool.busyCount)
+    s.ranges += this.perRange.size
+    s.groups += this.perGroup.size
+  }
+
   /** A line a minute: what the connections fetched, and how long a chunk took. */
   private report() {
     const seconds = (Date.now() - this.stats.since) / 1000
     if (seconds < 60) return
-    const { chunks, articles, fetchMs } = this.stats
+    const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
     if (chunks > 0) {
+      const ages = byAge
+        .map((b, i) =>
+          `${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
+            '– min',
+            '+ min'
+          )
+        )
+        .join(', ')
       logger.info(
         `Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
-          `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk, connections busy ${Math.round((fetchMs / (seconds * 1000 * NntpPool.size)) * 100)}%.`
+          `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
+          `hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
+          (this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus` : '') +
+          (this.stats.captured > 0 ? `; ${this.stats.captured} fetched into the corpus only` : '') +
+          '.'
+      )
+      const st = this.stats
+      const avg = (n: number) => (samples > 0 ? n / samples : 0)
+      const per = (ms: number) => (requests > 0 ? (ms / requests / 1000).toFixed(1) : '-')
+      logger.info(
+        `Connections: ${avg(st.inFlight).toFixed(1)} of ${NntpPool.size} with a command in flight (min ${st.inFlightMin}), ` +
+          `on ${avg(st.ranges).toFixed(1)} ranges of ${avg(st.groups).toFixed(1)} groups; ` +
+          `loops ${avg(st.fetching).toFixed(1)} fetching, ${avg(st.writing).toFixed(1)} writing, ${avg(st.idle).toFixed(1)} idle; ` +
+          `a request: GROUP ${per(st.groupMs)} s, XOVER reply ${per(st.replyMs)} s, transfer ${per(st.bodyMs)} s; ` +
+          `${st.silent} of ${requests} transfers silent >5 s; writing ${(st.writeMs / Math.max(1, chunks)).toFixed(0)} ms a chunk; ` +
+          `${st.deadlines} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB a minute); ${this.oldest()}.`
       )
     }
-    this.stats = { chunks: 0, articles: 0, fetchMs: 0, since: Date.now() }
+    this.stats = this.freshStats()
   }
 
   private peek(): boolean {
-    return this.targets.some((t) => t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
+    return this.targets.some(
+      (t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
+    )
   }
 
-  /** The first unclaimed missing chunk, in target order. */
+  /** Whether the target is under its range's and its group's limit of connections. */
+  private open(target: Target) {
+    const { perRange, perGroup } = coordinatorConfig.spool
+    return (
+      (perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
+      (perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
+    )
+  }
+
+  private count(target: Target, by: number) {
+    const range = (this.perRange.get(target.id) ?? 0) + by
+    const group = (this.perGroup.get(target.group) ?? 0) + by
+    if (range > 0) this.perRange.set(target.id, range)
+    else this.perRange.delete(target.id)
+    if (group > 0) this.perGroup.set(target.group, group)
+    else this.perGroup.delete(target.group)
+  }
+
+  /** The first unclaimed missing chunk, in target order, of a target under its limits. */
   private take(): { target: Target; span: ChunkSpan } | null {
     for (const target of this.targets) {
+      if (!this.open(target)) continue
       const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
       if (index === -1) continue
       const [span] = target.missing.splice(index, 1)
       this.claimed.add(`${target.id}:${span.from}`)
+      this.count(target, 1)
       return { target, span }
     }
     return null
@@ -133,28 +406,204 @@ export class Prefetcher {
       this.sizedAt = Date.now()
       this.spoolBytes = await Spool.size()
     }
-    const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
-    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id'))
+    const mine = (q: any) =>
+      this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
+    const rows = (q: any) =>
+      mine(
+        q
+          .from('ranges as r')
+          .join('groups as g', 'g.id', 'r.group_id')
+          .select(
+            'r.id',
+            'g.name as group',
+            'r.start_id',
+            'r.end_id',
+            'r.next_id',
+            'r.plain_chunks',
+            'r.capture'
+          )
+      )
     // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
     const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
+    // The groups workers hold (group_leases; REBUILD.md design 1), by lane: the prefetcher runs ahead of their
+    // cursors (a group's history and tail lanes are two cursors).
+    const held: { group: number; lane: string }[] = (
+      await db
+        .from('group_leases')
+        .whereNotNull('node_id')
+        .where(
+          'heartbeat_at',
+          '>=',
+          new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
+        )
+        .select('group_id', 'lane')
+    ).map((l) => ({ group: Number(l.group_id), lane: l.lane }))
     // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
-    const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
-    const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
-    const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
+    const ahead = Math.max(
+      coordinatorConfig.spool.minAhead,
+      coordinatorConfig.spool.aheadPerLease * Math.max(leased.length, held.length)
+    )
+    const started = await rows(db.query())
+      .where('r.status', 'planned')
+      .where('r.spooled_chunks', '>', 0)
+      .whereNull('r.spooled_at')
+      .orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
+    const [{ ready }] = await mine(
+      db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
+    ).count('* as ready')
     const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
-    const fresh = room && Number(ready) < ahead
-      ? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(ahead - Number(ready))
-      : []
+    // With at most perRange connections a range, enough ranges still being fetched for all loops: fully
+    // spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
+    // 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
+    const { perRange } = coordinatorConfig.spool
+    const short =
+      perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
+    const want = Math.max(ahead - Number(ready), short)
+    // The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
+    const full = await CatalogSpace.full()
+    // Ahead of each held cursor: the lane's next ranges in article order (the history lane's within what the group
+    // leases hand out: LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY, and none at the catalog's cap; the tail lane's
+    // at the tail's priority), a range of every held cursor before any cursor's second; without held groups (no
+    // window worker running) nothing is fetched ahead for workers.
+    const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
+    const tailPriority = coordinatorConfig.tail.priority
+    const heldHistory = full ? [] : held.filter((h) => h.lane === 'history').map((h) => h.group)
+    const heldTail = held.filter((h) => h.lane === 'tail').map((h) => h.group)
+    const fresh =
+      room && want > 0 && heldHistory.length + heldTail.length > 0
+        ? await rows(db.query())
+            .where('r.status', 'planned')
+            .where('r.spooled_chunks', 0)
+            .whereRaw('r.next_id <= r.end_id')
+            .where((q: any) =>
+              q
+                .where((h: any) =>
+                  h
+                    .whereIn('r.group_id', heldHistory)
+                    .where('r.priority', '>', tailPriority)
+                    .whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
+                    .where((c: any) => (leaseCaptureOnly ? c.where('r.capture', true) : c))
+                )
+                .orWhere((t: any) =>
+                  t.whereIn('r.group_id', heldTail).where('r.priority', '<=', tailPriority)
+                )
+            )
+            .orderByRaw(
+              'row_number() OVER (PARTITION BY r.group_id, r.priority <= ? ORDER BY r.start_id), r.priority, r.start_id',
+              [tailPriority]
+            )
+            .limit(want)
+        : []
     const targets: Target[] = []
     for (const row of [...leased, ...(room ? started : []), ...fresh]) {
       const start = Number(row.start_id)
       const end = Number(row.end_id)
       const total = Spool.chunksOf({ start, end }).length
       const missing: ChunkSpan[] = []
+      // A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
+      const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
       for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
-        if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
+        if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
+          continue
+        if (corpusRange && (await Corpus.has(corpusRange, span))) continue
+        missing.push(span)
+      }
+      if (missing.length > 0) {
+        targets.push({
+          id: Number(row.id),
+          group: row.group,
+          plain: new Set((row.plain_chunks ?? []).map(Number)),
+          missing,
+          total,
+          start,
+          end,
+          capture: row.capture === true,
+        })
+      }
+    }
+    // Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
+    for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
+      targets.push(target)
+    return targets
+  }
+
+  private filterRules: { at: number; filter: CorpusFilter } | null = null
+
+  /** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
+  private async corpusFilter(): Promise<CorpusFilter | undefined> {
+    if (!coordinatorConfig.corpus.filter) return undefined
+    if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
+      const rules = await RuleSets.current()
+      this.filterRules = {
+        at: Date.now(),
+        filter: { ruleVersion: rules.version, priorities: rules.priorities },
+      }
+    }
+    return this.filterRules.filter
+  }
+
+  /**
+   * Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
+   * (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
+   * as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
+   * took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
+   * a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
+   * ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
+   * group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
+   * gets stored_at and isn't read again.
+   */
+  private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
+    const { dir, ahead } = coordinatorConfig.corpus
+    if (!dir || ahead <= 0) return []
+    const { perRange, perGroup } = coordinatorConfig.spool
+    const ofGroup =
+      perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
+    const groups = new Map<string, number>()
+    const candidates = await rows(db.query())
+      .where('r.capture', true)
+      // A group taken off the list (groups.active) isn't worth the connections.
+      .where('g.active', true)
+      .whereNull('r.stored_at')
+      .where((q: any) =>
+        q
+          .where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
+          .orWhere('r.status', 'done')
+      )
+      .orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
+      .limit(ahead * 16)
+    const targets: Target[] = []
+    for (const row of candidates) {
+      if (targets.length >= ahead) break
+      const id = Number(row.id)
+      if (taken.has(id) || this.stored.has(id)) continue
+      if ((groups.get(row.group) ?? 0) >= ofGroup) continue
+      const start = Number(row.start_id)
+      const end = Number(row.end_id)
+      const range = { group: row.group, start, end }
+      if (await Corpus.complete(range)) {
+        this.stored.add(id)
+        await db.from('ranges').where('id', id).update({ stored_at: new Date() })
+        continue
+      }
+      const chunks = Spool.chunksOf({ start, end })
+      const missing: ChunkSpan[] = []
+      for (const span of chunks)
+        if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
+          missing.push(span)
+      if (missing.length > 0) {
+        targets.push({
+          id,
+          group: row.group,
+          plain: new Set((row.plain_chunks ?? []).map(Number)),
+          missing,
+          total: chunks.length,
+          start,
+          end,
+          capture: true,
+          corpusOnly: true,
+        })
+        groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
       }
-      if (missing.length > 0) targets.push({ id: Number(row.id), group: row.group, missing, total })
     }
     return targets
   }

app/services/prober.ts

@@ -0,0 +1,487 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { setTimeout as sleep } from 'node:timers/promises'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { YencError } from '#lib/yenc/index'
+import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
+import { NntpShare } from '#services/nntp_shares'
+import { DiskSpace } from '#services/disk_space'
+import { decidePar2Completeness } from '#services/par2_completeness'
+import { twinKey, twinNames } from '#services/hash_twins'
+import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
+import { PendingNamer } from '#services/pending_namer'
+
+interface Candidate {
+  id: number
+  releaseId: number
+  kind: string
+  messageId: string
+  /** A pending release's target (the pending ladder), not a catalog candidate: id and releaseId are 0. */
+  pending?: PendingProbe
+}
+
+type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
+
+/**
+ * The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
+ * par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones),
+ * then 7z listings (the end of a split 7z's last volume; queued from 2026-10-13, probed from 2026-10-23).
+ */
+const TIERS: { kind: string; unreadable: boolean }[] = [
+  { kind: 'lone', unreadable: false },
+  { kind: 'par2', unreadable: true },
+  { kind: 'archive', unreadable: true },
+  { kind: 'par2', unreadable: false },
+  { kind: 'archive', unreadable: false },
+  { kind: '7z-tail', unreadable: true },
+  { kind: '7z-tail', unreadable: false },
+]
+const FLUSH_MS = 2000
+/** Between sweeps that discard the waiting releases of dropped signatures. */
+const SWEEP_MS = 5 * 60_000
+/** A flush that deadlocks (against the batch store's pending upsert) keeps its outcomes for the next, this often. */
+const DEADLOCK_RETRIES = 5
+
+/**
+ * JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
+ * and the whole flush failed with it (LOG.md 2026-10-18 (78)).
+ */
+function jsonb(value: unknown): string {
+  return JSON.stringify(value, (_key, v) => (typeof v === 'string' ? v.replaceAll('\0', '') : v))
+}
+
+/** Rounds of every lane's pipeline still queued when the next claim starts. */
+const LOW_ROUNDS = 8
+
+/**
+ * The candidate queue's levels: the next claim starts below `low`, LOW_ROUNDS rounds of every lane's pipeline (at
+ * ~0.2 s an article and a pipeline of 8, ~13 s of work), and fills it to `target`, twice that. A claim takes 1.5–4 s
+ * and more (random slices of an 11 M-row table); refilling at half of four rounds left 3–6 s, and at 20–24 lanes the
+ * lanes waited for candidates up to 19% of their time (LOG.md 2026-10-28 (120)). At least `minBatch` either way.
+ */
+export function queueLevels(lanes: number, pipeline: number, minBatch: number): { low: number; target: number } {
+  const low = Math.max(minBatch, lanes * pipeline * LOW_ROUNDS)
+  return { low, target: 2 * low }
+}
+
+/** Slices a claim takes from random points of the queue. */
+const SLICES = 20
+const REPORT_MS = 60_000
+
+/**
+ * Phase 2's first step (REBUILD.md, part 6): reads the probe queue's
+ * articles (settled releases first, by kind) over PROBE_CONNECTIONS
+ * connections of its own, without compression, and stores what they show:
+ * per release and kind an Inspection, and on the release whether it's
+ * passworded and its inner name. One process: fetching (~0.1 s an article)
+ * dwarfs parsing (a few ms), so there's no queue between them.
+ */
+export class Prober {
+  private queue: Candidate[] = []
+  private outcomes: Outcome[] = []
+  private refilling: Promise<void> | null = null
+  private stopped = false
+  private stats = Prober.freshStats()
+  /** Flushes deadlocked in a row, their outcomes kept. */
+  private deadlocks = 0
+  /** Lanes running (each one connection, once it has work); at most the share's grant. Counted apart from the set: a lane leaving takes itself off at once. */
+  private lanes = new Set<Promise<void>>()
+  private running = 0
+  /** The last lane to look found nothing to claim: the prober wants its minimum only. */
+  private dry = false
+  /** What it stores (inspections, par2 file lists, the WAL) fills the NVMe: no claims below PROBE_MIN_FREE_GB. */
+  private space = new DiskSpace('probing', '/', coordinatorConfig.probe.minFreeGb)
+  private share = new NntpShare('prober', coordinatorConfig.nntpShares.proberPriority, coordinatorConfig.probe.minConnections, coordinatorConfig.probe.connections)
+
+  /** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
+  private static freshStats() {
+    return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, pending: 0, discarded: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
+  }
+  /** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
+  private namer = new PendingNamer()
+
+  async run() {
+    // A previous run's claims: one prober at a time.
+    const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
+    if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
+    const released = await PendingLadder.release()
+    if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
+    const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
+    // The ladder's discard sweep beside the lanes: a dropped signature's rows take minutes on the HDD (LOG.md (133)).
+    const sweeper = (async () => {
+      while (!this.stopped) {
+        try {
+          this.stats.discarded += await PendingLadder.discard()
+        } catch (error: any) {
+          logger.error(`Probing: the discard sweep failed: ${error.message}`)
+        }
+        for (let waited = 0; waited < SWEEP_MS && !this.stopped; waited += 1000) await sleep(1000)
+      }
+    })()
+    logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
+    const flusher = (async () => {
+      while (!this.stopped || this.outcomes.length > 0) {
+        await sleep(FLUSH_MS)
+        const flushed = Date.now()
+        await this.flush().catch((error) => logger.error(`Probing: storing results failed: ${error.message}`))
+        this.stats.flushMs += Date.now() - flushed
+        this.report()
+      }
+    })()
+    // Lanes as the share allows: the coordinator's history comes first (app/services/nntp_shares.ts).
+    while (!this.stopped) {
+      try {
+        const was = this.share.granted
+        const granted = await this.share.sync(this.dry ? 0 : this.share.max, this.running)
+        if (granted !== was) logger.info(`Probing: ${granted} connections (was ${was}).`)
+        while (this.running < granted) {
+          this.running++
+          const lane: Promise<void> = this.lane()
+            .catch((error) => logger.error(`Probing: a lane failed: ${error.message}`))
+            .finally(() => this.lanes.delete(lane))
+          this.lanes.add(lane)
+        }
+      } catch (error: any) {
+        logger.error(`Probing: NNTP share failed: ${error.message}`)
+      }
+      for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !this.stopped; waited += 1000) await sleep(1000)
+    }
+    await Promise.all(this.lanes)
+    await namer
+    await sweeper
+    await this.share.leave().catch(() => {})
+    // What was claimed and not probed goes back.
+    for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
+    this.stopped = true
+    await flusher
+  }
+
+  stop() {
+    this.stopped = true
+  }
+
+  /**
+   * One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure; done when
+   * there are more lanes than the share (its connection closed, the coordinator takes it on its next sync).
+   */
+  private async lane() {
+    let client: XoverClient | null = null
+    let usedAt = Date.now()
+    try {
+      while (!this.stopped) {
+        if (this.running > this.share.granted) break
+        const batch: Candidate[] = []
+        const waited = Date.now()
+        while (batch.length < coordinatorConfig.probe.pipeline) {
+          const candidate = await this.next()
+          if (!candidate) break
+          batch.push(candidate)
+        }
+        this.stats.waitMs += Date.now() - waited
+        this.dry = batch.length === 0
+        if (batch.length === 0) {
+          // Nothing to probe for a while: the connection closes (nntpIdleCloseMs), and opens again with work.
+          const idleMs = coordinatorConfig.nntpIdleCloseMs
+          if (client && idleMs > 0 && Date.now() - usedAt >= idleMs) {
+            client.close()
+            client = null
+          }
+          this.stats.idleMs += 30_000
+          await sleep(30_000)
+          continue
+        }
+        try {
+          client ??= await XoverClient.connect({
+            host: env.get('NNTP_HOST')!,
+            port: env.get('NNTP_PORT', 563),
+            user: env.get('NNTP_USER'),
+            password: env.get('NNTP_PASSWORD'),
+            secure: env.get('NNTP_SECURE', true),
+            compress: false,
+            timeoutMs: coordinatorConfig.nntpTimeoutMs,
+          })
+          const fetched = Date.now()
+          const bodies = await client.bodies(batch.map((c) => c.messageId))
+          this.stats.fetchMs += Date.now() - fetched
+          const inspected = Date.now()
+          for (const [i, candidate] of batch.entries()) {
+            const body = bodies[i]
+            this.stats.articles++
+            if (body === null) {
+              // Taken down or expired: final.
+              this.outcomes.push({ candidate, status: 'missing' })
+              continue
+            }
+            this.stats.bytes += body.length
+            try {
+              this.outcomes.push({ candidate, status: 'done', inspection: inspect(body, candidate.kind) })
+            } catch (error: any) {
+              // Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
+              if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
+              this.outcomes.push({ candidate, status: 'error' })
+            }
+          }
+          this.stats.inspectMs += Date.now() - inspected
+          usedAt = Date.now()
+        } catch (error: any) {
+          logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
+          for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
+          client?.close()
+          client = null
+          await sleep(5000)
+        }
+      }
+    } finally {
+      client?.close()
+      this.running--
+    }
+  }
+
+  /** The queue's low-water mark and the level a claim fills it to (queueLevels). */
+  private get levels() {
+    return queueLevels(this.running, coordinatorConfig.probe.pipeline, coordinatorConfig.probe.batch)
+  }
+
+  /** Claims to the queue's target; the claim after starts when it's below its low-water mark (queueLevels). */
+  private get batch() {
+    return this.levels.target
+  }
+
+  private async next(): Promise<Candidate | undefined> {
+    if (this.queue.length < this.levels.low) {
+      this.refilling ??= this.claim().finally(() => (this.refilling = null))
+      if (this.queue.length === 0) await this.refilling
+    }
+    return this.queue.shift()
+  }
+
+  /**
+   * Waiting candidates of settled releases, by kind, claimed so another round
+   * doesn't take them: SLICES slices from random points of the queue. In id
+   * order it went group by group, and some groups' articles take 0.6–3 s
+   * where most take 20–60 ms (a.b.zogg 2016: 7 articles/s on 6 connections).
+   */
+  private async claim() {
+    const started = Date.now()
+    try {
+      await this.claimTiers()
+    } finally {
+      this.stats.claimMs += Date.now() - started
+    }
+  }
+
+  private async claimTiers() {
+    // The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again.
+    if (await this.space.short()) return
+    await this.claimCatalog()
+    await this.claimPending()
+  }
+
+  /** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
+  private async claimPending() {
+    const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
+    const probes = await PendingLadder.claim(want)
+    if (probes.length === 0) return
+    const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
+      probes.map((p) => p.messageId),
+      PROBE_VERSION,
+    ])
+    const known = new Map(cached.map((c: any) => [c.message_id, c]))
+    for (const probe of probes) {
+      const candidate: Candidate = { id: 0, releaseId: 0, kind: probe.kind, messageId: probe.messageId, pending: probe }
+      const hit: any = known.get(probe.messageId)
+      if (hit) {
+        this.stats.cached++
+        this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
+      } else {
+        this.queue.push(candidate)
+      }
+    }
+  }
+
+  private async claimCatalog() {
+    for (const { kind, unreadable } of TIERS) {
+      const want = this.batch - this.queue.length
+      if (want <= 0) return
+      // The starting points chosen here: in the query Postgres drew random() once for all slices.
+      // Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
+      // Per value of unreadable, each an end of the index: over `unreadable >= false` Postgres walked every waiting
+      // entry of the kind, 2.5 minutes a claim at 28M archive candidates (LOG.md 2026-10-18 (74)).
+      const values = unreadable ? [true] : [false, true]
+      const ends = values
+        .map(() => `SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable = ?`)
+        .join(' UNION ALL ')
+      const [{ lo, hi }] = (
+        await db.rawQuery(
+          `SELECT min(lo) AS lo, max(hi) AS hi FROM (${ends}) e`,
+          values.flatMap((v) => [kind, v])
+        )
+      ).rows
+      if (lo === null) continue
+      const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
+      const { rows } = await db.rawQuery(
+        `UPDATE probe_candidates SET status = 'taken' WHERE id IN (
+           SELECT pc.id FROM unnest(?::bigint[]) AS o(start)
+           CROSS JOIN LATERAL (
+             SELECT pc.id FROM probe_candidates pc
+             WHERE pc.status = 'waiting' AND pc.ready AND pc.kind = ? AND pc.unreadable >= ? AND pc.id >= o.start
+             ORDER BY pc.kind, pc.unreadable, pc.id LIMIT ? FOR UPDATE SKIP LOCKED) pc)
+         RETURNING id, release_id, kind, message_id`,
+        [starts, kind, unreadable, Math.ceil(want / SLICES)]
+      )
+      const claimed = rows.map((r: any) => ({ id: Number(r.id), releaseId: Number(r.release_id), kind: r.kind, messageId: r.message_id }))
+      // Probed before (an earlier catalog's candidate of the same article): answered from probe_cache.
+      const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
+        claimed.map((c: Candidate) => c.messageId),
+        PROBE_VERSION,
+      ])
+      const known = new Map(cached.map((c: any) => [c.message_id, c]))
+      for (const candidate of claimed) {
+        const hit: any = known.get(candidate.messageId)
+        if (hit) {
+          this.stats.cached++
+          this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
+        } else {
+          this.queue.push(candidate)
+        }
+      }
+    }
+  }
+
+  /**
+   * The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. A deadlock
+   * (the batch store's pending upsert locks the same rows; LOG.md (134)) keeps the outcomes for the next flush
+   * instead of losing the probes.
+   */
+  private async flush() {
+    const taken = this.outcomes.splice(0)
+    if (taken.length === 0) return
+    try {
+      await this.store(taken)
+    } catch (error: any) {
+      if (error?.code === '40P01' && this.deadlocks < DEADLOCK_RETRIES) {
+        this.deadlocks++
+        this.outcomes.unshift(...taken)
+        logger.warn(`Probing: storing results deadlocked, kept for the next flush (${this.deadlocks}).`)
+        return
+      }
+      throw error
+    }
+    this.deadlocks = 0
+    for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
+    this.stats.pending += taken.filter((o) => o.candidate.pending && o.status !== 'waiting').length
+  }
+
+  private async store(taken: Outcome[]) {
+    // The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
+    const pending = taken.filter((o) => o.candidate.pending)
+    const outcomes = taken.filter((o) => !o.candidate.pending)
+    await db.transaction(async (trx) => {
+      if (outcomes.length > 0) {
+        await trx.rawQuery(
+          `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
+          [jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
+        )
+      }
+      await PendingLadder.record(pending.map((o) => ({ probe: o.candidate.pending!, status: o.status, inspection: o.inspection })), trx)
+      // Final answers read from the news server, kept by message-id for the next catalog (a reset).
+      // One per message-id: the same article is a candidate of more than one release, and ON CONFLICT can't take a
+      // row twice in one statement.
+      const fresh = [...new Map(taken.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
+      if (fresh.length > 0) {
+        await trx.rawQuery(
+          `INSERT INTO probe_cache (message_id, status, result, version)
+           SELECT v.message_id, v.status, v.result, ? FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, status text, result jsonb)
+           ON CONFLICT (message_id) DO UPDATE SET status = excluded.status, result = excluded.result, version = excluded.version, created_at = now()`,
+          [PROBE_VERSION, jsonb(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
+        )
+      }
+      const done = outcomes.filter((o) => o.inspection)
+      if (done.length === 0) return
+      await trx.rawQuery(
+        `INSERT INTO release_inspections (release_id, candidate_id, kind, result)
+         SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
+         ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
+        // A par2 set's file list goes to par2_files, not into every inspection.
+        [jsonb(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
+      )
+      await this.storeForMerge(trx, done)
+      // A par2 index read: the release's completeness by its set, when it has no counter.
+      await decidePar2Completeness(trx, [...new Set(done.filter((o) => o.inspection!.par2).map((o) => o.candidate.releaseId))])
+      // Each touched release from all its inspections (earlier rounds may have stored others).
+      const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
+      const { rows } = await trx.rawQuery(
+        `SELECT i.release_id, i.kind, i.result, r.name FROM release_inspections i JOIN releases r ON r.id = i.release_id WHERE i.release_id = ANY(?)`,
+        [ids]
+      )
+      const byRelease = new Map<number, { name: string; all: { kind: string; result: Inspection }[] }>()
+      for (const r of rows) {
+        const id = Number(r.release_id)
+        const entry = byRelease.get(id) ?? { name: r.name as string, all: [] as { kind: string; result: Inspection }[] }
+        entry.all.push({ kind: r.kind, result: r.result })
+        byRelease.set(id, entry)
+      }
+      // A twin by content names a release its own inspections didn't (hash_twins.ts; LOG.md (130)).
+      const twins = await twinNames(
+        done.filter((o) => o.inspection!.hash16k && o.inspection!.size).map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
+        trx
+      )
+      const twinOf = new Map<number, string>()
+      for (const o of done) {
+        const twin = o.inspection!.hash16k ? twins.get(twinKey(o.inspection!.hash16k, o.inspection!.size)) : undefined
+        if (twin && twin !== byRelease.get(o.candidate.releaseId)?.name) twinOf.set(o.candidate.releaseId, twin)
+      }
+      const updates = [...byRelease].map(([id, { name, all }]) => ({
+        id,
+        passworded: all.some((i) => i.result.passworded === true),
+        inner_name: innerName(all, name) ?? twinOf.get(id) ?? null,
+      }))
+      await trx.rawQuery(
+        `UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
+         FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
+        [jsonb(updates)]
+      )
+    })
+  }
+
+  /** What the par2 merge matches (app/services/par2_merger.ts): files' first-16-KiB hashes, par2 sets' file lists. */
+  private async storeForMerge(trx: any, done: Outcome[]) {
+    const hashed = done.filter((o) => o.inspection!.hash16k)
+    if (hashed.length > 0) {
+      await trx.rawQuery(
+        `INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
+         SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
+         ON CONFLICT (candidate_id) DO NOTHING`,
+        [jsonb(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
+      )
+    }
+    for (const o of done) {
+      const set = o.inspection!.par2
+      if (!set) continue
+      await trx.rawQuery(
+        `INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
+         SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
+         ON CONFLICT DO NOTHING`,
+        [set.setId, jsonb(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
+      )
+      await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
+    }
+  }
+
+  private report() {
+    const seconds = (Date.now() - this.stats.since) / 1000
+    if (seconds < REPORT_MS / 1000) return
+    const { articles, bytes, done, missing, error } = this.stats
+    logger.info(
+      `Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
+        `${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache; ` +
+        `the pending ladder ${this.stats.pending} probed, ${this.stats.discarded} discarded by signature. ` +
+        `Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
+        `idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
+        `claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`
+    )
+    this.stats = Prober.freshStats()
+  }
+}

app/services/release_judge.ts

@@ -0,0 +1,313 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { SOURCES } from '#lib/release/source'
+import { TINY_MIN_BYTES, NONSENSE_MIN_BYTES } from '#lib/rules/index'
+
+/** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */
+export const JUDGE_VERSION = 1
+/** Between rounds of the changed releases (judge_dirty, noted by triggers; migration 4900). */
+const ROUND_MS = 2 * 60 * 1000
+/**
+ * A full round (every release, each rule a scan of its table: 35–38 min) this often: for what no trigger notes. Time
+ * (unusable a day after settling, a release final once the ranges beside it are read) and fewer releases with a file
+ * after a merge (repeated).
+ */
+const FULL_MS = 24 * 60 * 60 * 1000
+/** Changed releases judged at a time. */
+const CHUNK = 50_000
+/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
+const REPEATED_RELEASES = 20
+
+/**
+ * A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. No "?" in
+ * these patterns: a raw query takes it for a binding ("h[ .]?26[45]" and "web[ .-]?dl" never matched; LOG.md
+ * 2026-10-16 (64)), so {0,1}.
+ */
+const MEDIA =
+  `(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]{0,1}26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]{0,1}dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
+/** The posters of scraped celebrity pictures. */
+const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
+/** An executable's name. */
+const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
+/**
+ * The tiny rule's name tests, lib/rules/rule_set.ts's RELEASE_LIKE_SPACED, EBOOK, TINY_CONTENT and
+ * STUB_EXEMPT_GROUPS in Postgres's dialect (and no "?", see MEDIA).
+ */
+const TINY_RELEASE_LIKE = `(\\y(19|20)[0-9][0-9]\\y|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-{0,1}dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\\.[0-9]+)+)\\y|-[a-z0-9]{2,}$|\\.[a-z0-9]+\\.[a-z0-9]+\\.)`
+const TINY_EBOOK = `\\y(epub|mobi|azw3{0,1}|pdf|djvu|cbr|cbz|e-{0,1}books{0,1})\\y`
+const TINY_CONTENT = `\\y(subs{0,1}|subtitles{0,1}|subpack|srt|untertitel|ondertitels{0,1}|ondertiteling|legendas{0,1}|napisy|sottotitoli|cracks{0,1}|cracked|keygen|keymaker|trainer|patch|patched|serials{0,1}|activator|loader|nocd|no-cd|fix|covers{0,1}|artwork|booklet|scans{0,1}|proof|sample|info|readme)\\y`
+const TINY_EXEMPT_GROUPS = `(sound|mp3|karaoke|music|audio|flac|lossless|e-{0,1}book|epub|comic|guitar|font|magazine)`
+
+export interface Rule {
+  name: string
+  /** Why, in a few words (the log, the web interface). */
+  says: string
+  /** SELECT of the release ids the rule judges; scoped, only those in judge_scope. */
+  sql: (scoped: boolean) => string
+}
+
+/** A join to the releases in scope (an incremental round), on the given release id column. */
+const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
+
+
+
+/**
+ * The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
+ * the header pass, the prober, the merges and the spots know by now.
+ */
+export const RULES: Rule[] = [
+  {
+    name: 'dead',
+    says: 'every probed first article is gone from the server (taken down or expired)',
+    sql: (scoped) =>
+      `SELECT c.release_id FROM probe_candidates c ${only(scoped, 'c.release_id')}
+       WHERE c.status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(c.status = 'missing')`,
+  },
+  {
+    name: 'fake',
+    says: 'named as a film, episode or music, but the content is a program',
+    // An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
+    // an executable by name or by its first bytes, under a media name.
+    sql: (scoped) => `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
+          WHERE i.kind IN ('archive', '7z-tail') AND jsonb_array_length(i.result->'names') > 0
+            AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
+            AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
+          UNION
+          SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
+          WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
+            AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
+  },
+  {
+    name: 'repeated',
+    says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
+    // Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
+    // (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
+    sql: (scoped) =>
+      scoped
+        ? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
+        : `SELECT DISTINCT h.release_id FROM file_hashes h
+          JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
+  },
+  {
+    name: 'scraped',
+    says: 'celebrity pictures scraped from the web, reposted by one spam family',
+    // "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
+    // a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
+    // titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
+    // By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the
+    // regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)).
+    sql: (scoped) =>
+      scoped
+        ? `SELECT r.id FROM releases r ${only(scoped, 'r.id')} JOIN posters p ON p.id = r.poster_id
+          WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`
+        : `SELECT r.id FROM posters p JOIN releases r ON r.poster_id = p.id
+          WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`,
+  },
+  {
+    name: 'locked',
+    says: 'passworded, and no spot, NZB or title gives the password',
+    // releases_passworded_idx: a scan of all of releases was 30 s a round.
+    sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
+  },
+  {
+    name: 'tiny',
+    says: 'one file of one article, 2–100 kB, under a release-like name: the exe-stub flood in miniature',
+    // The catalog's side of the edge's tiny-archive rule (lib/rules/rule_set.ts isTinyArchive; LOG.md (131),
+    // (135)): the same name tests in Postgres's regex dialect ("\y" for "\b", "{0,1}" for "?"), the readable flag
+    // for its scrambled and random tests, the group set for its groups. The 2015–17 flood that the edge indexed
+    // before the rule: 1.5M rows, 98% of them with an archive candidate, the rest lone par2 files and scrambled
+    // names, so no join on the candidates. A scan of releases (9 GB) a full round.
+    sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
+          WHERE r.files = 1 AND r.segments = 1 AND r.bytes >= ${TINY_MIN_BYTES} AND r.bytes < ${NONSENSE_MIN_BYTES}
+            AND r.readable AND NOT coalesce(r.sidecar_only, false)
+            AND r.name ~* '${TINY_RELEASE_LIKE}' AND r.name !~* '${TINY_EBOOK}' AND r.name !~* '${TINY_CONTENT}'
+            AND NOT EXISTS (SELECT 1 FROM group_sets gs WHERE gs.id = r.group_set_id AND gs.groups::text ~* '${TINY_EXEMPT_GROUPS}')`,
+  },
+  {
+    name: 'unusable',
+    says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
+    // Finished (settled_at): a provisional tail release's par2 or the rest of its files may still come.
+    sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
+          WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
+            AND r.settled_at IS NOT NULL
+            AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
+            AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
+            AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
+  },
+]
+
+export interface JudgeRound {
+  /** Releases judged: every one (a full round), or the changed ones and those a hash of theirs tipped over. */
+  releases?: number
+  /** Per rule: releases it judges now (in scope), verdicts it gained, verdicts it lost. */
+  rules: Record<string, { matched: number; added: number; removed: number }>
+}
+
+/**
+ * The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
+ * signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Every few
+ * minutes the releases whose facts changed (judge_dirty) are judged by every rule; a full round judges every release
+ * daily. A release gets the verdict of the first rule in order that matches it; one that no longer matches loses its
+ * verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
+ */
+export class ReleaseJudge {
+  private stopped = false
+
+  stop() {
+    this.stopped = true
+  }
+
+  async run() {
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        if (await this.fullDue()) {
+          const round = await this.full()
+          if (!this.stopped) logger.info(`Judged all releases: ${describe(round)}.`)
+        } else {
+          const round = await this.changed()
+          if (round.releases! > 0) logger.info(`Judged ${round.releases!.toLocaleString('en')} changed releases: ${describe(round)}.`)
+        }
+      } catch (error: any) {
+        logger.error(`Judge round failed: ${error.message}`)
+      }
+      for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  /** A full round is due: a day after the last, or the last was by another rule set. */
+  async fullDue(): Promise<boolean> {
+    const state = await db.from('judge_state').first()
+    return !state || state.version !== JUDGE_VERSION || Date.now() - new Date(state.full_at).getTime() >= FULL_MS
+  }
+
+  /**
+   * Every release by every rule, the matches gathered in judge_full (each rule its own statement: no transaction of
+   * half an hour), then applied at once. With dry, only counts. Stopped halfway, nothing is applied.
+   */
+  async full(dry = false): Promise<JudgeRound> {
+    const started = new Date()
+    const round: JudgeRound = { rules: {} }
+    // Verdicts of an older rule set are judged again from scratch.
+    if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
+    await db.rawQuery(`TRUNCATE judge_full`)
+    for (const rule of RULES) {
+      if (this.stopped) return round
+      const { rowCount } = await db.rawQuery(
+        `INSERT INTO judge_full (release_id, rule) SELECT id, ? FROM (${rule.sql(false)}) AS x(id) ON CONFLICT DO NOTHING`,
+        [rule.name]
+      )
+      round.rules[rule.name] = { matched: rowCount ?? 0, added: 0, removed: 0 }
+    }
+    if (dry || this.stopped) return round
+    await db.rawQuery(`ANALYZE judge_full`)
+    await db.transaction((trx) => this.apply(trx, 'judge_full', false, round))
+    await db.rawQuery(
+      `INSERT INTO judge_state (id, full_at, version) VALUES (1, ?, ?) ON CONFLICT (id) DO UPDATE SET full_at = excluded.full_at, version = excluded.version`,
+      [started, JUDGE_VERSION]
+    )
+    return round
+  }
+
+  /**
+   * The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
+   * A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
+   * already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
+   * file in that many without them had its releases judged before.
+   */
+  async changed(): Promise<JudgeRound> {
+    const round: JudgeRound = { releases: 0, rules: {} }
+    for (const rule of RULES) round.rules[rule.name] = { matched: 0, added: 0, removed: 0 }
+    let more = true
+    while (more && !this.stopped) {
+      await db.transaction(async (trx) => {
+        // Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
+        await trx.rawQuery(`DROP TABLE IF EXISTS judge_scope`)
+        await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
+        await trx.rawQuery(`CREATE TEMP TABLE judge_scope (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
+        const { rows } = await trx.rawQuery(
+          `WITH taken AS (DELETE FROM judge_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM judge_dirty LIMIT ${CHUNK})) RETURNING release_id),
+             scoped AS (INSERT INTO judge_scope SELECT DISTINCT release_id FROM taken ON CONFLICT DO NOTHING RETURNING 1)
+           SELECT (SELECT count(*) FROM taken) AS taken`
+        )
+        const taken = Number(rows[0].taken)
+        more = taken === CHUNK
+        if (taken === 0) return
+        // The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
+        await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
+        await trx.rawQuery(
+          `CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
+        )
+        await trx.rawQuery(
+          `INSERT INTO judge_tipped
+           SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
+           FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
+           CROSS JOIN LATERAL (
+             SELECT count(DISTINCT f.release_id) AS total,
+               count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
+             FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
+           WHERE c.total >= ${REPEATED_RELEASES}`
+        )
+        await trx.rawQuery(
+          `INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
+           WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
+           ON CONFLICT DO NOTHING`
+        )
+        await trx.rawQuery(`ANALYZE judge_scope`)
+        const { rows: scope } = await trx.rawQuery(`SELECT count(*) AS n FROM judge_scope`)
+        round.releases! += Number(scope[0].n)
+        await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL) ON COMMIT DROP`)
+        for (const rule of RULES) {
+          const { rowCount } = await trx.rawQuery(
+            `INSERT INTO judged (release_id, rule) SELECT id, ? FROM (${rule.sql(true)}) AS x(id) ON CONFLICT DO NOTHING`,
+            [rule.name]
+          )
+          round.rules[rule.name].matched += rowCount ?? 0
+        }
+        await trx.rawQuery(`ANALYZE judged`)
+        await this.apply(trx, 'judged', true, round)
+      })
+    }
+    return round
+  }
+
+  /**
+   * The matches (judged: release_id, rule; the first rule's each) into release_verdicts: verdicts no rule gives any
+   * more removed (scoped: of the releases in judge_scope only), changed ones written. Only the changes go through the
+   * insert: each match through its conflict check took 60–150 s a rule (LOG.md 2026-10-19 (80)).
+   */
+  private async apply(trx: any, judged: string, scoped: boolean, round: JudgeRound) {
+    const tally = (rows: { rule: string; n: string }[], field: 'added' | 'removed') => {
+      for (const { rule, n } of rows) {
+        round.rules[rule] ??= { matched: 0, added: 0, removed: 0 }
+        round.rules[rule][field] += Number(n)
+      }
+    }
+    const { rows: removed } = await trx.rawQuery(
+      `WITH gone AS (DELETE FROM release_verdicts v
+         WHERE ${scoped ? 'v.release_id IN (SELECT release_id FROM judge_scope) AND' : ''}
+           NOT EXISTS (SELECT 1 FROM ${judged} j WHERE j.release_id = v.release_id)
+         RETURNING v.rule)
+       SELECT rule, count(*) AS n FROM gone GROUP BY 1`
+    )
+    tally(removed, 'removed')
+    const { rows: added } = await trx.rawQuery(
+      `WITH written AS (INSERT INTO release_verdicts (release_id, rule, rule_version)
+         SELECT j.release_id, j.rule, ? FROM ${judged} j LEFT JOIN release_verdicts v ON v.release_id = j.release_id
+         WHERE v.rule IS DISTINCT FROM j.rule AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id)
+         ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
+         WHERE release_verdicts.rule <> excluded.rule
+         RETURNING rule)
+       SELECT rule, count(*) AS n FROM written GROUP BY 1`,
+      [JUDGE_VERSION]
+    )
+    tally(added, 'added')
+  }
+}
+
+/** "dead 1,157,018 (+1,136 −712), …" for the log. */
+function describe(round: JudgeRound): string {
+  return Object.entries(round.rules)
+    .map(([name, r]) => `${name} ${r.matched.toLocaleString('en')} (+${r.added.toLocaleString('en')} −${r.removed.toLocaleString('en')})`)
+    .join(', ')
+}

app/services/search_service.ts

@@ -0,0 +1,349 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import NzbService from '#services/nzb_service'
+import { SOURCES, sourceName } from '#lib/release/source'
+
+export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
+
+/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
+export interface SearchParams {
+  q?: string
+  /** Also releases the junk judge hid (release_verdicts). */
+  judged?: boolean
+  /** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
+  readable?: boolean
+  /** Only releases known to have all their files. */
+  complete?: boolean
+  /** Leave out releases known to miss files (Newznab). */
+  notIncomplete?: boolean
+  /** Leave out releases the prober found passworded (encrypted archive entries or headers). */
+  unlocked?: boolean
+  /** How the release was recognised: stem, subject or post. */
+  source?: string
+  poster?: string
+  /** Exact category ids (already expanded from parents). */
+  categories?: number[]
+  maxAgeDays?: number
+  /** TV: season and episode, matched as S01E02 or 1x02 in the name. */
+  season?: number
+  episode?: number
+  /** Size bounds in bytes. */
+  minSize?: number
+  maxSize?: number
+  sort?: SortOrder
+  page?: number
+  perPage?: number
+  /** Overrides `page` (Newznab pages by offset). */
+  offset?: number
+  /** Count matches up to this many past the page; `total` is capped there. */
+  countLimit?: number
+}
+
+/** A release in search results. */
+export interface ReleaseSummary {
+  id: number
+  name: string
+  source: string
+  poster: string | null
+  groups: string[]
+  postedAt: Date | null
+  fileTotal: number | null
+  fileCount: number
+  size: number
+  readable: boolean
+  complete: boolean | null
+  /** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */
+  final: boolean
+  category: number
+  settled: boolean
+  /** The prober found encrypted archive entries or headers; null: not probed (yet). */
+  passworded: boolean | null
+  /** A password is known (release_passwords): it's in the NZB. */
+  passwordKnown: boolean
+  /** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
+  innerName: string | null
+}
+
+/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
+const MAX_POSTERS = 200
+const SORTS: Record<SortOrder, string> = {
+  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
+  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
+  largest: 'r.bytes DESC, r.id DESC',
+  files: 'r.files DESC, r.id DESC',
+  name: 'r.name ASC, r.id ASC',
+}
+
+/** A broad search needn't count every match (each counted row is a heap read). */
+const COUNT_LIMIT = 10_000
+const STATS_TTL_MS = 5 * 60 * 1000
+/** A refresh that takes longer is given up (the next request tries again). */
+const STATS_TIMEOUT_MS = 2 * 60 * 1000
+
+/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
+function contains(word: string): string {
+  return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
+}
+
+/**
+ * A search word's condition on release_texts: anywhere in a text (ILIKE), or with a leading "^" at a word's start, the
+ * text's or after a character that's no letter or digit ("^zed" finds "Zed.Dead", "The_Zed", not "Facialized"). Both
+ * use the trigram index.
+ */
+function wordMatch(word: string): { sql: string; binding: string } {
+  if (word.startsWith('^') && word.length > 1) {
+    const rest = word.slice(1).replace(/[\\^$.|?*+()[\]{}]/g, (c) => `\\${c}`)
+    return { sql: 't.text ~* ?', binding: `(^|[^[:alnum:]])${rest}` }
+  }
+  return { sql: 't.text ILIKE ?', binding: contains(word) }
+}
+
+/** Searching and browsing the catalog, for the web interface and Newznab. */
+class SearchService {
+  async search(params: SearchParams) {
+    const where: string[] = ['r.files > 0']
+    // Judged junk (app/services/release_judge.ts) is left out unless asked for.
+    if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
+    const bindings: unknown[] = []
+    // Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its
+    // trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike;
+    // "^zed" only at a word's start.
+    for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
+      const match = wordMatch(word)
+      where.push(`r.id IN (SELECT t.release_id FROM release_texts t WHERE ${match.sql})`)
+      bindings.push(match.binding)
+    }
+    if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
+    if (params.complete) where.push('r.complete')
+    // Incomplete only when final: a provisional tail release may yet get its missing files.
+    if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)')
+    if (params.unlocked) where.push('r.passworded IS NOT TRUE')
+    if (params.source) {
+      const code = (SOURCES as Record<string, number>)[params.source]
+      if (code === undefined) where.push('false')
+      else {
+        where.push('r.source = ?')
+        bindings.push(code)
+      }
+    }
+    // A poster search: the matching posters first (posters, by trigram), each one's releases then by
+    // releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
+    // poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
+    let posters: number[] | null = null
+    if (params.poster) {
+      const { rows: matched } = await db.rawQuery(`SELECT id FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
+      posters = matched.map((m: { id: string }) => Number(m.id))
+    }
+    if (params.categories?.length) {
+      where.push('r.category = ANY(?)')
+      bindings.push(params.categories)
+    }
+    if (params.maxAgeDays) {
+      where.push('r.posted_at >= now() - make_interval(days => ?)')
+      bindings.push(params.maxAgeDays)
+    }
+    if (params.minSize !== undefined) {
+      where.push('r.bytes >= ?')
+      bindings.push(params.minSize)
+    }
+    if (params.maxSize !== undefined) {
+      where.push('r.bytes <= ?')
+      bindings.push(params.maxSize)
+    }
+    if (params.season !== undefined) {
+      const s = params.season
+      const e = params.episode
+      where.push('r.name ~* ?')
+      bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
+    }
+
+    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
+    const page = Math.max(params.page ?? 1, 1)
+    const offset = params.offset ?? (page - 1) * perPage
+    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
+    const whereSql = where.join(' AND ')
+    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
+    if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
+    // Per poster found: its releases matching everything else (releases_poster_newest_idx), the first offset + perPage
+    // in the order asked; then merged.
+    const source =
+      posters === null
+        ? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
+        : {
+            from: `unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
+                     SELECT r.* FROM releases r WHERE r.poster_id = p.id AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
+            where: 'true',
+            bindings: [posters] as unknown[],
+          }
+    const counting =
+      posters === null
+        ? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
+        : db.rawQuery(
+            `SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
+               SELECT 1 FROM releases r WHERE r.poster_id = p.id AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
+            [posters, ...bindings]
+          )
+    const [{ rows }, { rows: counted }] = await Promise.all([
+      db.rawQuery(
+        `SELECT r.id, r.name, r.source, po.poster, gs.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
+                EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
+                r.settled_at IS NOT NULL AS final
+         FROM ${source.from} LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id
+         WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
+        [...source.bindings, ...bindings, perPage, offset]
+      ),
+      counting,
+    ])
+    return {
+      page,
+      perPage,
+      offset,
+      total: Math.min(Number(counted[0].total), countLimit),
+      /** More than `total` match (the count stops at `countLimit`). */
+      totalCapped: Number(counted[0].total) > countLimit,
+      releases: (rows as any[]).map((r): ReleaseSummary => ({
+        id: Number(r.id),
+        name: r.name,
+        source: sourceName(r.source),
+        poster: r.poster ?? null,
+        groups: r.groups ?? [],
+        postedAt: r.posted_at,
+        fileTotal: r.file_total,
+        fileCount: Number(r.files),
+        size: Number(r.bytes),
+        readable: r.readable,
+        complete: r.complete,
+        final: r.final,
+        category: r.category,
+        settled: r.settled_at !== null,
+        passworded: r.passworded,
+        passwordKnown: r.password_known,
+        innerName: r.inner_name,
+      })),
+    }
+  }
+
+  /** A release with its files, read from its chunks. */
+  async release(id: number) {
+    const {
+      rows: [r],
+    } = await db.rawQuery(
+      `SELECT r.*, po.poster, gs.groups FROM releases r LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE r.id = ?`,
+      [id]
+    )
+    if (!r) return null
+    const [files, { rows: inspections }, passwords] = await Promise.all([
+      NzbService.files(id),
+      db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
+      NzbService.passwords(id),
+    ])
+    return {
+      id: Number(r.id),
+      name: r.name,
+      source: sourceName(r.source),
+      poster: r.poster ?? null,
+      groups: r.groups ?? [],
+      postedAt: r.posted_at,
+      fileTotal: r.file_total,
+      fileCount: Number(r.files),
+      size: Number(r.bytes),
+      segments: Number(r.segments),
+      readable: r.readable,
+      complete: r.complete,
+      final: r.settled_at !== null,
+      hasPar2: r.has_par2,
+      category: r.category,
+      firstSeenAt: r.first_seen_at,
+      settledAt: r.settled_at,
+      passworded: r.passworded,
+      /** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
+      passwords,
+      innerName: r.inner_name,
+      inspectedAt: r.inspected_at,
+      /** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
+      inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
+      files: files.map((f) => ({
+        name: f.name,
+        subject: f.subject,
+        poster: f.poster,
+        postedAt: f.postedAt === null ? null : new Date(f.postedAt),
+        size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
+        parts: f.segments.length,
+      })),
+    }
+  }
+
+  /** Totals for the web interface, computed at most once a minute (callers in between share them). */
+  async stats() {
+    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
+      this.statsRefresh ??= Promise.race([
+        this.computeStats(),
+        new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()),
+      ])
+        .then((value) => {
+          this.statsCache = { at: Date.now(), value }
+        })
+        .catch((error) => {
+          logger.warn(`Stats: ${error.message}`)
+          if (!this.statsCache) throw error
+        })
+        .finally(() => {
+          this.statsRefresh = undefined
+        })
+      if (!this.statsCache) await this.statsRefresh
+    }
+    return this.statsCache!.value
+  }
+
+  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
+  private statsRefresh?: Promise<void>
+
+  /**
+   * Estimates (a catalog of 100M+ rows can't be counted on every visit):
+   * the row count from Postgres's statistics, below a million rows exact;
+   * readable share, files and size from a sample of 5,000 rows; ranges
+   * counted exactly (a few hundred thousand rows).
+   */
+  private async computeStats() {
+    const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
+    const estimate = Number(rows[0]?.n ?? 0)
+    // A fixed-size sample: a percentage of a big catalog is many random reads on the HDD.
+    const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)'
+    const { rows: s } = await db.rawQuery(
+      `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
+       FROM releases ${sample} WHERE files > 0`
+    )
+    const sampled = Number(s[0].sampled)
+    const releases = sample ? Math.max(estimate, 0) : sampled
+    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
+    // History by articles; the pass: the history ranges before those deferred behind them (priority
+    // 2000 apart; e.g. groups added later that go first).
+    const { rows: ranges } = await db.rawQuery(
+      `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0 AND status <> 'ancient'),
+            pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
+       SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges WHERE status <> 'ancient') AS total,
+              coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
+              coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
+              coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
+              (SELECT max(done_at) FROM ranges) AS last_done_at,
+              (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
+       FROM h`
+    )
+    const g = ranges[0]
+    return {
+      approximate: sample !== '',
+      releases,
+      readableReleases: scale(s[0].readable),
+      files: scale(s[0].files),
+      size: scale(s[0].bytes),
+      ranges: { done: Number(g.done), total: Number(g.total) },
+      /** History and the current pass, in articles. */
+      history: { done: Number(g.history_done), total: Number(g.history) },
+      pass: { done: Number(g.pass_done), total: Number(g.pass) },
+      lastIndexedAt: g.last_done_at,
+      tailIndexedAt: g.tail_done_at,
+    }
+  }
+}
+
+export default new SearchService()

app/services/settler.ts

@@ -1,27 +1,35 @@
 import db from '@adonisjs/lucid/services/db'
 
-/** Releases unchanged for this long are settled (ranges are read in order; the merger adds within minutes). */
-const QUIET_HOURS = 6
-const BATCH = 50_000
+/** Hours a complete provisional release stays open for its window's continuation before it is settled here. */
+export const SETTLE_COMPLETE_HOURS = 6
 
 /**
- * Settles releases (REBUILD.md, part 8): once nothing has been added for
- * QUIET_HOURS, its completeness is decided: complete when every file the
- * [i/N] counter announced was found, unknown without a counter (phase 2
- * reads par2 sets for those). Phase 2 picks its work from settled releases.
+ * Provisional releases (REBUILD.md, "Voyager 3: window indexing", 2): a release the window hadn't closed on at a
+ * lease's end is written with `settled_at` null, for the window's continuation to finish. A continuation may
+ * never come (the range-lease path of the first run left 109k such rows, LOG.md (118); a group's next lease is
+ * hours away), and the catalog shows the release as still collecting. A complete one — every file its counter
+ * announced found — can't gain a counted file any more, so once it has been quiet for SETTLE_COMPLETE_HOURS it is
+ * settled here, as the finish would: `settled_at` set (the judge's trigger fires on it), its waiting probe
+ * candidates made claimable. Incomplete ones keep waiting: the continuation adds their files.
  */
 class Settler {
-  async run(): Promise<number> {
-    let settled = 0
-    while (true) {
-      const { rowCount } = await db.rawQuery(
-        `UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
-         WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(hours => ?) LIMIT ?)`,
-        [QUIET_HOURS, BATCH]
+  async settleComplete(quietHours = SETTLE_COMPLETE_HOURS): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows } = await trx.rawQuery(
+        `UPDATE releases SET settled_at = now()
+         WHERE settled_at IS NULL AND complete
+           AND first_seen_at < now() - make_interval(hours => ?) AND last_posted_at < now() - make_interval(hours => ?)
+         RETURNING id`,
+        [quietHours, quietHours]
       )
-      settled += rowCount ?? 0
-      if ((rowCount ?? 0) < BATCH) return settled
-    }
+      if (rows.length > 0)
+        await trx.rawQuery(
+          `UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
+           WHERE pc.release_id = ANY(?::bigint[]) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
+          [rows.map((r: { id: string }) => Number(r.id))]
+        )
+      return rows.length
+    })
   }
 }
 

app/services/seven_zip_tails.ts

@@ -0,0 +1,22 @@
+/**
+ * 7z keeps its listing at the end of the last volume, so a split 7z gets a `7z-tail` probe candidate: the last
+ * segment of its highest volume (lib/edge/range_indexer.ts). RAR lists its files in every volume's header, and
+ * its first volume's start (one `archive` candidate) is enough. The live tail reads an upload in pieces while
+ * it's posted, and each piece queues a tail for the last volume it saw so far (TsIgNBewuFbuwWy: 7, for
+ * .002–.008; LOG.md 2026-10-23 (94)); merges bring their members' tails together. Of the given releases'
+ * waiting tails, those below the release's highest volume go. Those of the highest volume stay, all of them:
+ * a piece that saw only part of it queued a segment that isn't its last.
+ */
+export async function pruneSevenZipTails(trx: any, releaseIds: (number | string)[]): Promise<number> {
+  if (releaseIds.length === 0) return 0
+  const result = await trx.rawQuery(
+    `DELETE FROM probe_candidates pc USING (
+       SELECT id, status, volume, max(volume) OVER (PARTITION BY release_id) AS highest
+       FROM (SELECT id, release_id, status, substring(file from '\\.7z\\.(\\d{1,4})$')::int AS volume
+             FROM probe_candidates WHERE release_id = ANY(?) AND kind = '7z-tail') t
+     ) s
+     WHERE pc.id = s.id AND s.status = 'waiting' AND s.volume < s.highest`,
+    [releaseIds.map(String)]
+  )
+  return result.rowCount ?? 0
+}

app/services/sidecar_folder.ts

@@ -0,0 +1,89 @@
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
+
+/** Candidates looked at per round. */
+const BATCH = 2000
+/** An upload's sidecar is posted with it: its release is looked for this far from it. */
+const WINDOW_MINUTES = 10
+/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
+export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
+
+/** The words of a name worth comparing: 4+ letters or digits. */
+function words(text: string): Set<string> {
+  return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4))
+}
+
+export interface Sibling {
+  id: number
+  name: string
+  files: number
+  file_total: number | null
+}
+
+/**
+ * Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject
+ * ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one
+ * under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that
+ * isn't one release.
+ */
+export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null {
+  const said = words(files.map((f) => f.subject).join(' '))
+  const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w)))
+  if (named.length === 1) return named[0]
+  if (named.length > 1) return null
+  const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : []
+  return counted.length === 1 ? counted[0] : null
+}
+
+export interface FoldRound {
+  looked: number
+  sidecars: number
+  folded: number
+}
+
+/**
+ * The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another
+ * counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small
+ * settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in
+ * the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via
+ * 'sidecar'). The rest leave the queue untouched.
+ */
+export class SidecarFolder {
+  constructor(private merge: (trx: any, found: Map<string, any>, key: string, via: 'sidecar') => Promise<number>) {}
+
+  async round(): Promise<FoldRound> {
+    const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
+    // A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole
+    // queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)).
+    const { rows: queued } = await db.rawQuery(
+      `SELECT f.release_id AS queued_id, r.id, r.group_id, r.poster_id, r.posted_at, r.files, r.file_total, r.readable
+       FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
+      [BATCH]
+    )
+    const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id))
+    for (const c of queued) {
+      if (c.id === null) continue
+      round.looked++
+      const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
+      const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
+      if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
+      round.sidecars++
+      if (!c.posted_at) continue
+      const { rows: siblings } = await db.rawQuery(
+        `SELECT r.id, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
+         WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
+           AND r.group_id = ? AND r.poster_id = ? AND r.id <> ? AND r.files > ?`,
+        [c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster_id, c.id, c.files]
+      )
+      const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
+      if (!sibling) continue
+      const full = siblings.find((s: any) => Number(s.id) === sibling.id)
+      const merged = await db.transaction((trx) =>
+        this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar')
+      )
+      if (merged > 0) round.folded++
+    }
+    if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete()
+    return round
+  }
+}

app/services/spot_inspector.ts

@@ -0,0 +1,381 @@
+import { createHash } from 'node:crypto'
+import { gzipSync } from 'node:zlib'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { decodeSpotBinary } from '#lib/spotnet/index'
+import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
+import { parseSubject } from '#lib/headers/index'
+import { releaseIdentity } from '#lib/release/index'
+import { identityKey, releaseKey } from '#lib/release/keys'
+import { SOURCES } from '#lib/release/source'
+import CatalogRefs from '#services/catalog_refs'
+
+/** Spots claimed per round, newest first. */
+const BATCH = 20
+/** Between rounds when there was nothing to do. */
+const IDLE_MS = 60_000
+/** Unmatched spots are matched again this often (the header pass may have built their releases since). */
+const REMATCH_MS = 60 * 60 * 1000
+/**
+ * Unmatched spots matched again per hour: after a reset every spot is (16.5k at 500 an hour took 33 h). Never tried
+ * first, then the newest: the oldest inspection first tried a 3-month backfill before today's spots, the ones whose
+ * releases the tail has (2 of 16.7k matched 3 h after reset 24).
+ */
+const REMATCH_BATCH = 5000
+/** Unmatched spots are matched again for this long after their inspection. */
+const REMATCH_DAYS = 30
+/** A file's release posted further apart than this from the NZB's date is another upload of the same name. */
+const MAX_SPREAD_MS = 48 * 3600 * 1000
+
+export interface InspectRound {
+  inspected: number
+  matched: number
+  standalone: number
+  outcomes: Record<string, number>
+}
+
+interface SpotFile {
+  idx: number
+  name: string
+  subject: string
+  poster: string
+  groups: string[]
+  postedAt: Date | null
+  bytes: number
+  segments: number
+}
+
+/**
+ * A placeholder NZB (Dummy.rar + Dummy.nzb, shared by many spots): under 1% of the size the spot announces, or under
+ * 16 KB in all (text-only spots). Its files aren't listed (voyager 1's isDummyNzb).
+ */
+export function isDummyNzb(nzb: ParsedNzb, spotSize: number): boolean {
+  const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
+  return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
+}
+
+/** A file of an NZB as the header pass names it: the subject's file name, by the same parser. */
+export function nzbFileName(subject: string): { name: string; base: string } {
+  const parsed = parseSubject(subject) ?? parseSubject(`${subject} (1/1)`)
+  return parsed ? { name: parsed.name, base: parsed.base } : { name: subject, base: subject }
+}
+
+/**
+ * The spot inspector (REBUILD.md, spots): looks inside each listed spot, on
+ * one NNTP connection of its own. It reads the spot's compressed NZB (its
+ * articles in alt.binaries.ftd), keeps the list of files in `spot_files`,
+ * and matches every file to the release the header pass built for it: the
+ * release identity of the file's subject and poster (lib/release, as the
+ * edge workers compute it), in one of the file's groups, posted within 48 h.
+ * Each matched release is linked in `release_spots`, and gets the spot's
+ * password in `release_passwords` (the NZB's own `<meta type="password">`
+ * first, else what the spot's title or text names) and `passworded`. A spot
+ * whose files match nothing is matched again every hour, for 30 days; once
+ * SPOT_STANDALONE_AFTER_HOURS have passed since its inspection it becomes a
+ * release of its own (source `spot`) with its NZB, until the header pass
+ * builds the releases after all (a spot older than what the catalog has
+ * indexed): then it moves to those, and its own release goes.
+ */
+export class SpotInspector {
+  private stopped = false
+  private client?: XoverClient
+  private groupIds = new Map<string, number | null>()
+  private rematchedAt = 0
+
+  stop() {
+    this.stopped = true
+    this.client?.close()
+  }
+
+  async run() {
+    while (!this.stopped) {
+      let round: InspectRound | null = null
+      try {
+        round = await this.round()
+        if (round.inspected > 0 || round.standalone > 0) {
+          const outcomes = Object.entries(round.outcomes).map(([k, n]) => `${n} ${k}`).join(', ')
+          logger.info(`Spots inspected: ${round.inspected} (${outcomes}); ${round.matched} matched releases, ${round.standalone} of their own.`)
+        }
+      } catch (error: any) {
+        logger.error(`Spot inspection failed: ${error.message}`)
+        this.client?.close()
+        this.client = undefined
+      }
+      if (!round || round.inspected === 0) {
+        // Nothing to inspect for a while: closed (nntpIdleCloseMs), opened again with work.
+        const idleMs = coordinatorConfig.nntpIdleCloseMs
+        if (idleMs > 0 && IDLE_MS >= idleMs) {
+          this.client?.close()
+          this.client = undefined
+        }
+        for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+      }
+    }
+  }
+
+  private async connect(): Promise<XoverClient> {
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: false,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    return this.client
+  }
+
+  async round(): Promise<InspectRound> {
+    const round: InspectRound = { inspected: 0, matched: 0, standalone: 0, outcomes: {} }
+    const spots = await db.from('spots').where('status', 'listed').orderBy('posted_at', 'desc').limit(BATCH)
+    for (const spot of spots) {
+      if (this.stopped) break
+      const outcome = await this.inspect(spot)
+      round.inspected++
+      round.outcomes[outcome.status] = (round.outcomes[outcome.status] ?? 0) + 1
+      round.matched += outcome.matched
+    }
+    if (Date.now() - this.rematchedAt >= REMATCH_MS && !this.stopped) {
+      this.rematchedAt = Date.now()
+      const again = await this.rematch()
+      round.matched += again.matched
+      round.standalone += again.standalone
+    }
+    return round
+  }
+
+  /** One spot: its NZB read, its files listed and matched. */
+  async inspect(spot: any): Promise<{ status: string; matched: number }> {
+    const done = async (status: string, extra: Record<string, unknown> = {}) => {
+      await db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
+      return { status, matched: 0 }
+    }
+    const segments: string[] = spot.nzb_segments ?? []
+    if (segments.length === 0) return done('no-nzb')
+    if (segments.length > coordinatorConfig.spots.maxNzbSegments) return done('no-nzb', { error: `${segments.length} NZB articles` })
+    const client = await this.connect()
+    const bodies = await client.bodies(segments)
+    if (bodies.some((b) => b === null)) return done('nzb-gone', { error: `${bodies.filter((b) => b === null).length} of ${segments.length} NZB articles gone` })
+    let xml: string
+    try {
+      xml = decodeSpotBinary(bodies as Buffer[]).toString('utf8')
+    } catch (error: any) {
+      return done('nzb-unreadable', { error: String(error.message).slice(0, 500) })
+    }
+    return this.inspectNzb(spot, xml)
+  }
+
+  /** The files of a spot's NZB listed and matched (the tests feed this directly). */
+  async inspectNzb(spot: any, xml: string): Promise<{ status: string; matched: number }> {
+    const nzb = parseNzb(xml)
+    const finish = (status: string, extra: Record<string, unknown> = {}) =>
+      db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
+    if (!nzb || nzb.files.length === 0) {
+      await finish('no-nzb')
+      return { status: 'no-nzb', matched: 0 }
+    }
+    if (isDummyNzb(nzb, Number(spot.size ?? 0))) {
+      await finish('dummy')
+      return { status: 'dummy', matched: 0 }
+    }
+    const files: SpotFile[] = nzb.files.map((f, idx) => ({
+      idx,
+      name: nzbFileName(f.subject).name.slice(0, 1000),
+      subject: f.subject.slice(0, 2000),
+      poster: f.poster,
+      groups: f.groups,
+      postedAt: f.date ? new Date(f.date * 1000) : null,
+      bytes: f.segments.reduce((s, x) => s + x.bytes, 0),
+      segments: f.segments.length,
+    }))
+    // The NZB's own password over what the spot's text names.
+    const password = nzb.meta.password ? { password: nzb.meta.password.slice(0, 200), source: 'nzb' } : spot.password ? { password: spot.password, source: spot.password_source } : null
+    await db.transaction(async (trx) => {
+      await trx.from('spot_files').where('spot_id', spot.message_id).delete()
+      for (let i = 0; i < files.length; i += 1000) {
+        await trx.table('spot_files').multiInsert(
+          files.slice(i, i + 1000).map((f) => ({
+            spot_id: spot.message_id,
+            idx: f.idx,
+            name: f.name,
+            subject: f.subject,
+            poster: f.poster,
+            groups: f.groups,
+            posted_at: f.postedAt,
+            bytes: f.bytes,
+            segments: f.segments,
+          }))
+        )
+      }
+      await trx
+        .from('spots')
+        .where('message_id', spot.message_id)
+        .update({
+          status: 'inspected',
+          inspected_at: new Date(),
+          files: files.length,
+          matched: 0,
+          nzb: gzipSync(xml),
+          ...(password && password.source === 'nzb' ? { password: password.password, password_source: 'nzb' } : {}),
+        })
+    })
+    const matched = await this.match(spot.message_id)
+    return { status: 'inspected', matched }
+  }
+
+  private async groupId(name: string): Promise<number | null> {
+    if (!this.groupIds.has(name)) {
+      const row = await db.from('groups').where('name', name).select('id').first()
+      this.groupIds.set(name, row ? Number(row.id) : null)
+    }
+    return this.groupIds.get(name)!
+  }
+
+  /**
+   * Matches a spot's listed files to releases and links them; how many releases. A file's release: the same
+   * identity (releaseIdentity over its subject and poster) in one of its groups, the one posted nearest the file.
+   */
+  async match(spotId: string): Promise<number> {
+    const spot = await db.from('spots').where('message_id', spotId).first()
+    if (!spot || spot.status !== 'inspected') return 0
+    const files = await db.from('spot_files').where('spot_id', spotId).orderBy('idx')
+    const wanted: { idx: number; key: string; groups: number[]; at: number | null }[] = []
+    for (const f of files) {
+      if (!f.subject) continue
+      const { name, base } = nzbFileName(f.subject)
+      const identity = releaseIdentity({ poster: f.poster ?? '', subject: base, name, weak: false })
+      if (!identity) continue
+      const groups = (await Promise.all((f.groups ?? []).map((g: string) => this.groupId(g)))).filter((g): g is number => g !== null)
+      if (groups.length > 0) wanted.push({ idx: f.idx, key: identity.key, groups, at: f.posted_at ? new Date(f.posted_at).getTime() : null })
+    }
+    if (wanted.length === 0) return 0
+    const { rows } = await db.rawQuery(
+      `SELECT id, encode(identity_key, 'hex') AS identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?::bytea[]) AND group_id = ANY(?)`,
+      [[...new Set(wanted.map((w) => w.key))].map((k) => identityKey(k)), [...new Set(wanted.flatMap((w) => w.groups))]]
+    )
+    const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
+    for (const r of rows) {
+      const list = byKey.get(r.identity_key) ?? []
+      list.push({ id: Number(r.id), group: Number(r.group_id), at: r.posted_at ? new Date(r.posted_at).getTime() : null })
+      byKey.set(r.identity_key, list)
+    }
+    const fileRelease = new Map<number, number>()
+    for (const w of wanted) {
+      const near = (byKey.get(w.key) ?? [])
+        .filter((r) => w.groups.includes(r.group))
+        .map((r) => ({ ...r, spread: r.at === null || w.at === null ? 0 : Math.abs(r.at - w.at) }))
+        .filter((r) => r.spread <= MAX_SPREAD_MS)
+        .sort((a, b) => a.spread - b.spread)
+      if (near[0]) fileRelease.set(w.idx, near[0].id)
+    }
+    const perRelease = new Map<number, number>()
+    for (const id of fileRelease.values()) perRelease.set(id, (perRelease.get(id) ?? 0) + 1)
+    if (perRelease.size === 0) return 0
+    await db.transaction(async (trx) => {
+      // A release of the spot's own goes now that the header pass built its files' releases.
+      if (spot.release_id) {
+        const own = await trx.from('releases').where('id', spot.release_id).where('source', SOURCES.spot).select('id').first()
+        if (own) {
+          await trx.from('release_spots').where('release_id', own.id).delete()
+          await trx.from('release_passwords').where('release_id', own.id).delete()
+          await trx.from('releases').where('id', own.id).delete()
+        }
+      }
+      for (const [idx, releaseId] of fileRelease) await trx.from('spot_files').where({ spot_id: spotId, idx }).update({ release_id: releaseId })
+      for (const [releaseId, n] of perRelease) {
+        await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: n }).onConflict(['release_id', 'spot_id']).merge({ files: n })
+      }
+      if (spot.password && perRelease.size > 0) {
+        await trx
+          .table('release_passwords')
+          .multiInsert([...perRelease.keys()].map((id) => ({ release_id: id, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId })))
+          .onConflict(['release_id', 'password'])
+          .ignore()
+        await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
+      }
+      // The spot's category for the ones whose names say nothing (migration 5200: spot_category).
+      await trx.rawQuery(`UPDATE releases SET category = c FROM (SELECT spot_category(?::integer, ?::jsonb) AS c) s WHERE id = ANY(?) AND category = 8010 AND c IS NOT NULL`, [
+        spot.category,
+        JSON.stringify(spot.subcats ?? {}),
+        [...perRelease.keys()],
+      ])
+      const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
+      // The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
+      await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
+    })
+    return perRelease.size
+  }
+
+  /** Unmatched inspected spots matched again; the ones that waited long enough become releases of their own. */
+  async rematch(): Promise<{ matched: number; standalone: number }> {
+    const out = { matched: 0, standalone: 0 }
+    const { rows: spots } = await db.rawQuery(
+      `SELECT message_id, inspected_at, release_id FROM spots
+       WHERE status = 'inspected' AND matched = 0 AND inspected_at > now() - make_interval(days => ?)
+       ORDER BY rematched_at NULLS FIRST, posted_at DESC NULLS LAST LIMIT ?`,
+      [REMATCH_DAYS, REMATCH_BATCH]
+    )
+    const cutoff = Date.now() - coordinatorConfig.spots.standaloneAfterHours * 3600 * 1000
+    for (const spot of spots) {
+      if (this.stopped) break
+      const n = await this.match(spot.message_id)
+      await db.from('spots').where('message_id', spot.message_id).update({ rematched_at: new Date() })
+      out.matched += n
+      if (n === 0 && !spot.release_id && new Date(spot.inspected_at).getTime() < cutoff && (await this.standalone(spot.message_id))) out.standalone++
+    }
+    return out
+  }
+
+  /** A release of the spot's own (source `spot`), from its listed files; false when it has none. */
+  async standalone(spotId: string): Promise<boolean> {
+    const spot = await db.from('spots').where('message_id', spotId).first()
+    const files = await db.from('spot_files').where('spot_id', spotId)
+    if (!spot || files.length === 0) return false
+    const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
+    const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
+    const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
+    // Its poster and group set by id (voyager 3, migration 6000), as the batch store stores its releases'.
+    const posterId = spot.poster ? await CatalogRefs.posterId(db, spot.poster) : null
+    const groupSetId = await CatalogRefs.groupSetId(db, groups)
+    const [row] = await db
+      .table('releases')
+      .insert({
+        release_key: releaseKey(key, groupId, null),
+        identity_key: identityKey(key),
+        group_id: groupId,
+        group_set_id: groupSetId,
+        poster_id: posterId,
+        name: spot.title.slice(0, 1000),
+        source: SOURCES.spot,
+        posted_at: spot.posted_at,
+        last_posted_at: spot.posted_at,
+        files: files.length,
+        segments: files.reduce((s: number, f: any) => s + Number(f.segments), 0),
+        bytes: files.reduce((s: number, f: any) => s + Number(f.bytes), 0),
+        readable: true,
+        passworded: spot.password ? true : null,
+        settled_at: new Date(),
+        // Kept when its title says nothing (the category trigger, migration 5200).
+        category: db.raw('coalesce(spot_category(?::integer, ?::jsonb), 8010)', [spot.category, JSON.stringify(spot.subcats ?? {})]),
+      })
+      .onConflict('release_key')
+      .ignore()
+      .returning('id')
+    if (!row) return false
+    const releaseId = Number(row.id)
+    await db.transaction(async (trx) => {
+      await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
+      // Its title, searchable (release_texts).
+      await trx.rawQuery(`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`, [releaseId, spot.title])
+      if (spot.password) {
+        await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
+      }
+      await trx.from('spots').where('message_id', spotId).update({ release_id: releaseId })
+    })
+    return true
+  }
+}

app/services/spotter.ts

@@ -0,0 +1,191 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { SOURCES } from '#lib/release/source'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { MissingArticle, XoverClient } from '#lib/nntp/index'
+import {
+  decodeSpotText,
+  moderationTarget,
+  parseSpotArticleHeaders,
+  parseSpotHeader,
+  parseSpotXml,
+  passwordFromSpot,
+  spotterId,
+  verifySpotHeader,
+} from '#lib/spotnet/index'
+
+/** Overviews asked for at a time. */
+const XOVER_STEP = 1000
+/** A personal dispose counts within this long of the spot (Spotweb). */
+const DISPOSE_WINDOW_SECONDS = 5 * 86400
+
+export interface SpotterRound {
+  read: number
+  listed: number
+  unverified: number
+  moderation: number
+}
+
+/**
+ * The spotter (REBUILD.md, spots): Spotnet's spots, from the headers of
+ * free.pt into `spots`, on one NNTP connection of its own. Every few
+ * minutes it reads the overviews after the last article it saw (the first
+ * time the last SPOT_BACKFILL), and for each spot the full header: its
+ * signature is checked there (lib/spotnet, as Spotweb does), and the X-XML
+ * gives the description, the website and the NZB's articles. A password the
+ * title (`{{pw}}`) or the description names is kept with the spot; the spot
+ * inspector carries it to the releases. A delete/dispose message marks its
+ * target `removed` and takes back what the target gave the releases.
+ */
+export class Spotter {
+  private stopped = false
+  private client?: XoverClient
+
+  stop() {
+    this.stopped = true
+    this.client?.close()
+  }
+
+  async run() {
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        const round = await this.round()
+        if (round.read > 0) {
+          logger.info(`Spots: ${round.read} articles read, ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
+        }
+      } catch (error: any) {
+        logger.error(`Spot round failed: ${error.message}`)
+        this.client?.close()
+        this.client = undefined
+      }
+      const wait = coordinatorConfig.spots.everySeconds * 1000
+      // Quiet until the next round: closed (nntpIdleCloseMs), opened again then.
+      const idleMs = coordinatorConfig.nntpIdleCloseMs
+      if (idleMs > 0 && wait - (Date.now() - started) >= idleMs) {
+        this.client?.close()
+        this.client = undefined
+      }
+      for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  private async connect(): Promise<XoverClient> {
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: false,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    return this.client
+  }
+
+  /** The articles after the last one seen, up to the group's high. */
+  async round(): Promise<SpotterRound> {
+    const { group, backfill } = coordinatorConfig.spots
+    const client = await this.connect()
+    const { low, high } = await client.group(group)
+    const state = await db.from('spot_state').where('group_name', group).first()
+    let next = state ? Number(state.last_article) + 1 : Math.max(low, high - backfill + 1)
+    const round: SpotterRound = { read: 0, listed: 0, unverified: 0, moderation: 0 }
+    while (next <= high && !this.stopped) {
+      const to = Math.min(high, next + XOVER_STEP - 1)
+      const overviews = await client.xover(next, to)
+      round.read += overviews.length
+      for (const [number, fields] of overviews) {
+        if (this.stopped) return round
+        const messageId = (fields['message-id'] ?? '').replace(/^<|>$/g, '')
+        if (!messageId || !parseSpotHeader(fields.from ?? '', fields.subject ?? '', messageId, fields.date)) continue
+        const outcome = await this.read(client, messageId, number)
+        if (outcome) round[outcome]++
+      }
+      await db
+        .table('spot_state')
+        .insert({ group_name: group, last_article: to })
+        .onConflict('group_name')
+        .merge({ last_article: to })
+      next = to + 1
+    }
+    return round
+  }
+
+  /** One spot by its full header; what it was, or null when it was there already or is gone. */
+  async read(client: XoverClient, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
+    if (await db.from('spots').where('message_id', messageId).first()) return null
+    let raw: Buffer
+    try {
+      raw = await client.head(messageId)
+    } catch (error) {
+      if (error instanceof MissingArticle) return null
+      throw error
+    }
+    return this.store(raw.toString('latin1'), messageId, articleNumber)
+  }
+
+  /** Verifies and stores a spot from its raw header (the tests feed this directly). */
+  async store(raw: string, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
+    const { headers, xml: xmlText } = parseSpotArticleHeaders(raw)
+    const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
+    if (!header) return null
+    const verified = verifySpotHeader(header)
+    const xml = verified ? parseSpotXml(decodeSpotText(xmlText)) : null
+    const target = verified ? moderationTarget(header) : null
+    const title = xml?.title || header.title
+    // Kept apart by where it came from: the title's {{pw}}, else the description's "Wachtwoord: …".
+    const fromTitle = verified && !target ? passwordFromSpot(title, null) : null
+    const fromText = verified && !target && !fromTitle ? passwordFromSpot('', xml?.description) : null
+    const status = !verified ? 'unverified' : target ? 'moderation' : 'listed'
+    await db
+      .table('spots')
+      .insert({
+        message_id: messageId,
+        article_number: articleNumber,
+        title: title.slice(0, 2000),
+        tag: header.tag || null,
+        poster: header.poster,
+        category: header.category,
+        subcats: JSON.stringify(header.subcats),
+        size: header.size || null,
+        posted_at: header.stamp ? new Date(header.stamp * 1000) : null,
+        key_id: header.keyId,
+        spotter_id: header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : null,
+        description: xml?.description || null,
+        website: xml?.website || null,
+        nzb_segments: xml?.nzbSegments.length ? xml.nzbSegments : null,
+        password: fromTitle ?? fromText,
+        password_source: fromTitle ? 'spot-title' : fromText ? 'spot-text' : null,
+        status,
+      })
+      .onConflict('message_id')
+      .ignore()
+    if (target) await this.moderate(header.selfSignedPubKey, header.stamp, target)
+    return status
+  }
+
+  /**
+   * A delete/dispose/remove: the target is `removed`, and its links and passwords go from the releases. As in
+   * Spotweb, a personal dispose (the poster's own key) counts only from the same spotter within 5 days; a
+   * moderator's (Spotnet's key) always.
+   */
+  private async moderate(pubKey: string, stamp: number, targetId: string) {
+    const target = await db.from('spots').where('message_id', targetId).first()
+    if (!target) return
+    const personal = pubKey.length > 50 ? spotterId(pubKey) : ''
+    if (personal) {
+      const age = stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
+      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
+    }
+    await db.transaction(async (trx) => {
+      await trx.from('spots').where('message_id', targetId).update({ status: 'removed' })
+      await trx.from('release_spots').where('spot_id', targetId).delete()
+      await trx.from('release_passwords').where('spot_id', targetId).delete()
+      // A release the spot made of its own goes with it.
+      if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', SOURCES.spot).delete()
+    })
+    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
+  }
+}

app/services/status.ts

@@ -0,0 +1,343 @@
+import db from '@adonisjs/lucid/services/db'
+import { readdir, stat, statfs } from 'node:fs/promises'
+import { join } from 'node:path'
+import coordinatorConfig from '#config/coordinator'
+import GroupLeases from '#services/group_leases'
+import type { Lane } from '#lib/edge/index'
+
+export interface RangeBand {
+  band: string
+  planned: number
+  plannedArticles: number
+  leased: number
+  done: number
+  doneArticles: number
+  failed: number
+}
+
+export interface HeldGroup {
+  node: string
+  group: string
+  lane: Lane
+  range: { start: number; end: number; next: number } | null
+  heartbeatSeconds: number
+  snapshotNext: number | null
+  snapshotSeconds: number | null
+  snapshotMb: number | null
+  rangesDone: number
+  headers: number
+}
+
+export interface SnapshotFile {
+  group: string
+  lane: Lane
+  mb: number
+  hours: number
+}
+
+export interface Status {
+  at: string
+  catalog: {
+    tablespace: string
+    gb: number
+    capGb: number
+    full: boolean
+    freeGb: number
+  }
+  bands: RangeBand[]
+  lastHour: {
+    indexedArticles: number
+    doneRanges: number
+    storedArticles: number
+    storedRanges: number
+  }
+  corpus: { pendingRanges: number; pendingArticles: number }
+  spool: { ranges: number; chunks: number }
+  tail: { groups: number; ranges: number; articles: number }
+  /** The 'ancient' ranges: articles below the low watermark the groups were planned at (expired, never fetched). */
+  ancient: { groups: number; articles: number }
+  held: HeldGroup[]
+  snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
+  releases: { estimated: number; provisional: number }
+  /** `ready` and `waiting` are the planner's estimates (the last analyze), the others counts. */
+  probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
+  judge: { dirty: number }
+  nodes: { name: string; lastSeenSeconds: number | null }[]
+}
+
+const seconds = (since: Date | string | null) =>
+  since ? Math.round((Date.now() - new Date(since).getTime()) / 1000) : null
+
+/**
+ * What the indexer is doing, in one object (commands/coordinator_status.ts prints it): the catalog's room on the
+ * NVMe, the ranges by band (the tail, the capture windows, the history), the last hour's rates, the held groups
+ * with their snapshots, the phase-2 backlogs. Every number comes from the database, the snapshot directory or the
+ * filesystem; nothing is kept between calls.
+ */
+class StatusService {
+  async collect(): Promise<Status> {
+    const tail = coordinatorConfig.tail.priority
+    const cap = coordinatorConfig.catalogMaxNvmeGb
+    // The catalog's tablespace only (CatalogSpace measures it every minute anyway): pg_database_size walks the
+    // chunk partitions and the probe cache on the HDD, minutes on homelab.
+    const { rows: space } = await db.rawQuery(
+      `SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
+              pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
+    )
+    const tablespaceGb = Number(space[0].bytes) / 1e9
+    const fs = await statfs('/').catch(() => null)
+    const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
+
+    // The ranges by band: the tail, the capture windows (priorities 0–2; the sampling ranges in other years are
+    // capture too, at their history priorities), the history by its thousand.
+    const { rows: bands } = await db.rawQuery(
+      `SELECT CASE WHEN priority <= ? THEN 'tail'
+                   WHEN priority < 0 THEN 'raised'
+                   WHEN capture AND priority <= 2 THEN 'window ' || priority
+                   WHEN priority < 1000 THEN 'history 0-999'
+                   ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
+              count(*) FILTER (WHERE status = 'planned') AS planned,
+              coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned'), 0) AS planned_articles,
+              count(*) FILTER (WHERE status = 'leased') AS leased,
+              count(*) FILTER (WHERE status = 'done') AS done,
+              coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
+              count(*) FILTER (WHERE status = 'failed') AS failed
+       FROM ranges GROUP BY 1
+       ORDER BY min(CASE WHEN priority <= ? THEN 0 WHEN priority < 0 THEN 1 WHEN capture AND priority <= 2 THEN 2 + priority
+                        WHEN priority < 1000 THEN 10 ELSE 11 + priority / 1000 END)`,
+      [tail, tail]
+    )
+    const { rows: hour } = await db.rawQuery(
+      `SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
+              count(*) FILTER (WHERE done_at > now() - interval '1 hour') AS done_ranges,
+              coalesce(sum(end_id - start_id + 1) FILTER (WHERE stored_at > now() - interval '1 hour'), 0) AS stored,
+              count(*) FILTER (WHERE stored_at > now() - interval '1 hour') AS stored_ranges,
+              count(*) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done') AS corpus_pending,
+              coalesce(sum(end_id - start_id + 1) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done'), 0) AS corpus_pending_articles,
+              count(*) FILTER (WHERE status IN ('planned', 'leased') AND spooled_chunks > 0) AS spooled_ranges,
+              coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
+              count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
+              count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
+              coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles,
+              count(DISTINCT group_id) FILTER (WHERE status = 'ancient') AS ancient_groups,
+              coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'ancient'), 0) AS ancient_articles
+       FROM ranges`,
+      [tail, tail, tail]
+    )
+    const h = hour[0]
+
+    const ttl = coordinatorConfig.leaseTtlSeconds
+    const { rows: held } = await db.rawQuery(
+      `SELECT n.name AS node, g.name AS "group", l.lane, l.heartbeat_at, l.snapshot_next, l.snapshot_at, l.ranges_done, l.headers,
+              r.start_id, r.end_id, r.next_id, l.group_id
+       FROM group_leases l JOIN groups g ON g.id = l.group_id JOIN nodes n ON n.id = l.node_id
+       LEFT JOIN ranges r ON r.id = l.range_id
+       WHERE l.node_id IS NOT NULL AND l.heartbeat_at >= now() - make_interval(secs => ?)
+       ORDER BY l.lane, g.name`,
+      [ttl]
+    )
+    const snapshotSizes = new Map<string, number>()
+    const files: SnapshotFile[] = []
+    const names = new Map<number, string>(
+      (await db.from('groups').select('id', 'name')).map((g) => [Number(g.id), g.name as string])
+    )
+    for (const file of await readdir(GroupLeases.dir).catch(() => [] as string[])) {
+      const m = /^(\d+)(-tail)?\.gz$/.exec(file)
+      if (!m) continue
+      const s = await stat(join(GroupLeases.dir, file)).catch(() => null)
+      if (!s) continue
+      const lane: Lane = m[2] ? 'tail' : 'history'
+      snapshotSizes.set(`${m[1]}:${lane}`, s.size)
+      files.push({
+        group: names.get(Number(m[1])) ?? m[1],
+        lane,
+        mb: s.size / 1e6,
+        hours: (Date.now() - s.mtimeMs) / 3600_000,
+      })
+    }
+    files.sort((a, b) => b.mb - a.mb)
+
+    // The probe backlog from the planner's estimates of the partial indexes (counting 11 M rows took 10 s on
+    // homelab); the rest are small partial indexes.
+    const { rows: releases } = await db.rawQuery(
+      `SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
+              (SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
+              (SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_ready_idx'::regclass) AS ready,
+              (SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_waiting_idx'::regclass) AS waiting,
+              (SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
+              (SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
+              (SELECT count(*) FROM judge_dirty) AS dirty`
+    )
+    const r = releases[0]
+    const nodes = await db.from('nodes').select('name', 'last_seen_at').orderBy('name')
+
+    return {
+      at: new Date().toISOString(),
+      catalog: {
+        tablespace: space[0].tablespace,
+        gb: tablespaceGb,
+        capGb: cap,
+        full: cap > 0 && tablespaceGb > cap,
+        freeGb,
+      },
+      bands: bands.map((b: any) => ({
+        band: b.band,
+        planned: Number(b.planned),
+        plannedArticles: Number(b.planned_articles),
+        leased: Number(b.leased),
+        done: Number(b.done),
+        doneArticles: Number(b.done_articles),
+        failed: Number(b.failed),
+      })),
+      lastHour: {
+        indexedArticles: Number(h.indexed),
+        doneRanges: Number(h.done_ranges),
+        storedArticles: Number(h.stored),
+        storedRanges: Number(h.stored_ranges),
+      },
+      corpus: {
+        pendingRanges: Number(h.corpus_pending),
+        pendingArticles: Number(h.corpus_pending_articles),
+      },
+      spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
+      ancient: { groups: Number(h.ancient_groups), articles: Number(h.ancient_articles) },
+      tail: {
+        groups: Number(h.tail_groups),
+        ranges: Number(h.tail_ranges),
+        articles: Number(h.tail_articles),
+      },
+      held: held.map((l: any) => {
+        const size = snapshotSizes.get(`${l.group_id}:${l.lane}`)
+        return {
+          node: l.node,
+          group: l.group,
+          lane: l.lane,
+          range:
+            l.start_id !== null
+              ? { start: Number(l.start_id), end: Number(l.end_id), next: Number(l.next_id) }
+              : null,
+          heartbeatSeconds: seconds(l.heartbeat_at) ?? 0,
+          snapshotNext: l.snapshot_next === null ? null : Number(l.snapshot_next),
+          snapshotSeconds: seconds(l.snapshot_at),
+          snapshotMb: size === undefined ? null : size / 1e6,
+          rangesDone: Number(l.ranges_done),
+          headers: Number(l.headers),
+        }
+      }),
+      snapshots: {
+        count: files.length,
+        gb: files.reduce((sum, f) => sum + f.mb, 0) / 1000,
+        oldestHours: files.length > 0 ? Math.max(...files.map((f) => f.hours)) : null,
+        largest: files.slice(0, 5),
+      },
+      releases: { estimated: Math.max(0, Number(r.estimated)), provisional: Number(r.provisional) },
+      probes: {
+        ready: Number(r.ready),
+        waiting: Number(r.waiting),
+        pendingWaiting: Number(r.pending_waiting),
+        pendingNamed: Number(r.pending_named),
+      },
+      judge: { dirty: Number(r.dirty) },
+      nodes: nodes.map((n) => ({ name: n.name, lastSeenSeconds: seconds(n.last_seen_at) })),
+    }
+  }
+
+  /** The status as a text panel. */
+  render(s: Status): string {
+    const n = (v: number) => Math.round(v).toLocaleString('en-US')
+    const b = (v: number) =>
+      v >= 1e9
+        ? `${(v / 1e9).toFixed(2)} B`
+        : v >= 1e6
+          ? `${(v / 1e6).toFixed(1)} M`
+          : v >= 1e3
+            ? `${(v / 1e3).toFixed(0)} k`
+            : String(v)
+    const age = (sec: number | null) =>
+      sec === null
+        ? '-'
+        : sec < 90
+          ? `${sec} s`
+          : sec < 5400
+            ? `${Math.round(sec / 60)} min`
+            : `${(sec / 3600).toFixed(1)} h`
+    const lines: string[] = []
+    lines.push(`Status at ${s.at}`)
+    lines.push('')
+    lines.push(
+      `Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; ${s.catalog.freeGb.toFixed(1)} GB free on /`
+    )
+    lines.push(
+      `Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
+    )
+    lines.push(
+      `Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups. Ancient: ${b(s.ancient.articles)} articles below the low watermark of ${n(s.ancient.groups)} groups`
+    )
+    lines.push('')
+    const table = (header: string[], rows: string[][]) => {
+      const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
+      const line = (r: string[]) =>
+        r.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join('  ')
+      lines.push(line(header))
+      for (const r of rows) lines.push(line(r))
+    }
+    lines.push('Ranges by band:')
+    table(
+      ['band', 'planned', 'articles', 'leased', 'done', 'articles', 'failed'],
+      s.bands.map((x) => [
+        x.band,
+        n(x.planned),
+        b(x.plannedArticles),
+        n(x.leased),
+        n(x.done),
+        b(x.doneArticles),
+        n(x.failed),
+      ])
+    )
+    lines.push('')
+    lines.push(`Held groups (${s.held.length}):`)
+    if (s.held.length > 0)
+      table(
+        [
+          'group',
+          'lane',
+          'node',
+          'range',
+          'at',
+          'heartbeat',
+          'snapshot',
+          'size',
+          'done',
+          'headers',
+        ],
+        s.held.map((x) => [
+          x.group,
+          x.lane,
+          x.node,
+          x.range ? `${x.range.start}-${x.range.end}` : '-',
+          x.range ? String(x.range.next) : '-',
+          age(x.heartbeatSeconds),
+          x.snapshotNext === null ? '-' : `${x.snapshotNext} (${age(x.snapshotSeconds)})`,
+          x.snapshotMb === null ? '-' : `${x.snapshotMb.toFixed(0)} MB`,
+          n(x.rangesDone),
+          b(x.headers),
+        ])
+      )
+    lines.push('')
+    lines.push(
+      `Snapshots: ${n(s.snapshots.count)} files, ${s.snapshots.gb.toFixed(2)} GB${s.snapshots.oldestHours === null ? '' : `, oldest ${s.snapshots.oldestHours.toFixed(1)} h`}${s.snapshots.largest.length > 0 ? `; largest ${s.snapshots.largest.map((f) => `${f.group}${f.lane === 'tail' ? ' (tail)' : ''} ${f.mb.toFixed(0)} MB`).join(', ')}` : ''}`
+    )
+    lines.push(
+      `Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
+    )
+    lines.push(
+      `Probes: ~${n(s.probes.ready)} ready of ~${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
+    )
+    lines.push(
+      `Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`
+    )
+    return lines.join('\n')
+  }
+}
+
+export default new StatusService()

commands/coordinator_run.ts

@@ -3,34 +3,46 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
 import { setTimeout as sleep } from 'node:timers/promises'
 import { Prefetcher } from '#services/prefetcher'
 import Planner from '#services/planner'
-import FragmentMerger from '#services/fragment_merger'
+import Leases from '#services/leases'
 import Settler from '#services/settler'
 import NntpPool from '#services/nntp_pool'
+import { NntpShare } from '#services/nntp_shares'
+import coordinatorConfig from '#config/coordinator'
 
 const TICK_MS = 2000
 const PLAN_EVERY_MS = 60 * 60 * 1000
-const MERGE_EVERY_MS = 60 * 1000
-const MERGE_BUDGET_MS = 30 * 1000
 const STOP_WAIT_MS = 15 * 1000
+/** Leases whose worker went quiet after reading everything are finished every few minutes. */
+const ABANDONED_EVERY_MS = 5 * 60 * 1000
 
 /**
- * The coordinator's background work (REBUILD.md, part 1): the prefetcher,
- * merging fragments of finished ranges every minute, and every hour
- * planning ranges for new articles and settling quiet releases. The API (leases, stream,
- * batches) runs in the web server.
+ * The coordinator's background work (REBUILD.md, part 1): the prefetcher, the tail's ranges every few minutes,
+ * new groups' history every hour, and abandoned leases. The API (leases, stream, batches) runs in the web server.
+ * Voyager 3: the settler and the fragment merger are gone (the window finishes a release once; the block merger
+ * joins edge releases).
  */
 export default class CoordinatorRun extends BaseCommand {
   static commandName = 'coordinator:run'
   static description = 'Prefetches ranges into the spool and plans new ones'
   static options: CommandOptions = { startApp: true, staysAlive: true }
 
-  @flags.string({ description: 'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles' })
+  @flags.string({
+    description:
+      'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles',
+  })
   declare shard?: string
 
   async run() {
     const [index, count] = (this.shard ?? '1/1').split('/').map(Number)
-    if (!(count >= 1 && index >= 1 && index <= count)) throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
+    if (!(count >= 1 && index >= 1 && index <= count))
+      throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
     const prefetcher = new Prefetcher({ index: index - 1, count })
+    const share = new NntpShare(
+      count > 1 ? `coordinator ${index}/${count}` : 'coordinator',
+      coordinatorConfig.nntpShares.coordinatorPriority,
+      coordinatorConfig.nntpMinConnections,
+      coordinatorConfig.nntpConnections
+    )
     const main = index === 1
     let stopping = false
     let stopped: () => void
@@ -42,9 +54,33 @@ export default class CoordinatorRun extends BaseCommand {
       prefetcher.stop()
       await done
     })
-    this.logger.info(count > 1 ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).` : 'Coordinator started.')
-    // Two loops: the prefetcher's ticks don't wait for merging or planning (a merge round
-    // of up to MERGE_BUDGET_MS left the connections idle once their targets were fetched).
+    this.logger.info(
+      count > 1
+        ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).`
+        : 'Coordinator started.'
+    )
+    // The pool's size: its share of the account's connections, the prober the rest (app/services/nntp_shares.ts).
+    // Before the first connection: until then it's NNTP_CONNECTIONS.
+    const first = await NntpPool.share(share)
+    this.logger.info(`NNTP share: ${first.granted} connections.`)
+    const sharing = (async () => {
+      while (!stopping) {
+        try {
+          const { was, granted } = await NntpPool.share(share)
+          if (granted !== was) this.logger.info(`NNTP share: ${granted} connections (was ${was}).`)
+        } catch (error: any) {
+          this.logger.error(`NNTP share failed: ${error.message}`)
+        }
+        for (
+          let waited = 0;
+          waited < coordinatorConfig.nntpShares.everyMs && !stopping;
+          waited += TICK_MS
+        )
+          await sleep(TICK_MS)
+      }
+    })()
+    // The prefetcher's ticks in a loop of their own: planning doesn't hold them (a merge round of up to 30 s once
+    // left the connections idle once their targets were fetched).
     const prefetching = (async () => {
       while (!stopping) {
         try {
@@ -55,47 +91,59 @@ export default class CoordinatorRun extends BaseCommand {
         await sleep(TICK_MS)
       }
     })()
+    const abandoned = (async () => {
+      while (main && !stopping) {
+        try {
+          const finished = await Leases.finishAbandoned()
+          if (finished > 0)
+            this.logger.info(
+              `Finished ${finished} ranges read to their end whose worker went quiet.`
+            )
+        } catch (error: any) {
+          this.logger.error(`Finishing abandoned leases failed: ${error.message}`)
+        }
+        try {
+          const settled = await Settler.settleComplete()
+          if (settled > 0)
+            this.logger.info(`Settled ${settled} complete provisional releases quiet for hours.`)
+        } catch (error: any) {
+          this.logger.error(`Settling complete releases failed: ${error.message}`)
+        }
+        for (let waited = 0; waited < ABANDONED_EVERY_MS && !stopping; waited += TICK_MS)
+          await sleep(TICK_MS)
+      }
+    })()
+    // History for new groups hourly; the tail every few minutes.
     let plannedAt = 0
-    let mergedAt = 0
+    let tailedAt = 0
     while (!stopping) {
       try {
         if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
           plannedAt = Date.now()
           const { groups, ranges } = await Planner.planAll()
           if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
-          const settled = await Settler.run()
-          if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
         }
-        if (main && Date.now() - mergedAt >= MERGE_EVERY_MS) {
-          mergedAt = Date.now()
-          // Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
-          // (Going on while only waiting files were left kept Postgres busy all the time.)
-          const totals = { completed: 0, incomplete: 0, waiting: 0 }
-          let decidedThisPass = 0
-          while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
-            const merged = await FragmentMerger.run()
-            totals.completed += merged.completed
-            totals.incomplete += merged.incomplete
-            totals.waiting = merged.waiting
-            decidedThisPass += merged.completed + merged.incomplete
-            if (merged.wrapped) {
-              if (decidedThisPass === 0) break
-              decidedThisPass = 0
-            }
-          }
-          if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
+        if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
+          tailedAt = Date.now()
+          const { groups, ranges, extended } = await Planner.planTail()
+          if (ranges + extended > 0)
+            this.logger.info(
+              `Tail: ${ranges} ranges added, ${extended} grown, for ${groups} groups.`
+            )
         }
       } catch (error: any) {
         this.logger.error(`Coordinator round failed: ${error.message}`)
       }
       await sleep(TICK_MS)
     }
-    await prefetching
+    await Promise.all([prefetching, abandoned, sharing])
     // The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
     const waitUntil = Date.now() + STOP_WAIT_MS
     while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
-    if (prefetcher.active > 0) this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
+    if (prefetcher.active > 0)
+      this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
     await NntpPool.close()
+    await share.leave().catch(() => {})
     this.logger.info('Coordinator stopped.')
     stopped!()
     await this.terminate()

commands/coordinator_status.ts

@@ -0,0 +1,22 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import StatusService from '#services/status'
+
+/**
+ * What the indexer is doing (app/services/status.ts): the catalog's room, the ranges by band, the last hour's
+ * rates, the held groups and their snapshots, the phase-2 backlogs. Reads only.
+ */
+export default class CoordinatorStatus extends BaseCommand {
+  static commandName = 'coordinator:status'
+  static description =
+    'Prints what the indexer is doing: catalog room, ranges by band, rates, held groups, backlogs'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'JSON instead of the text panel' })
+  declare json: boolean
+
+  async run() {
+    const status = await StatusService.collect()
+    this.logger.log(this.json ? JSON.stringify(status, null, 2) : StatusService.render(status))
+  }
+}

commands/corpus_filter.ts

@@ -0,0 +1,54 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { access, readFile, readdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import db from '@adonisjs/lucid/services/db'
+import Corpus, { FILTERED } from '#services/corpus'
+import RuleSets from '#services/rule_sets'
+import coordinatorConfig from '#config/coordinator'
+
+/**
+ * Filters corpus chunks stored as the news server sent them (before CORPUS_FILTER, 2026-10-23): each chunk without a
+ * `.drops` of the planned capture ranges (--all: of every range in the corpus) written again as Corpus.write would.
+ */
+export default class CorpusFilterCommand extends BaseCommand {
+  static commandName = 'corpus:filter'
+  static description = 'Filters stored corpus chunks (planned capture ranges; --all: every range) as CORPUS_FILTER does'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'Every range in the corpus, also done ones (the grouping sweeps read them raw)' })
+  declare all: boolean
+
+  async run() {
+    const dir = coordinatorConfig.corpus.dir
+    if (!dir) return this.logger.error('CORPUS_DIR is not set')
+    const rules = await RuleSets.current()
+    const filter = { ruleVersion: rules.version, priorities: rules.priorities }
+    const query = db.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').where('r.capture', true).select('g.name as group', 'r.start_id', 'r.end_id')
+    const ranges = await (this.all ? query : query.where('r.status', 'planned'))
+    let chunks = 0
+    let before = 0
+    let after = 0
+    for (const r of ranges) {
+      const range = { group: r.group, start: Number(r.start_id), end: Number(r.end_id) }
+      const rangeDir = join(dir, range.group, `${range.start}-${range.end}`)
+      const files = await readdir(rangeDir).catch(() => [] as string[])
+      const gz = files.filter((f) => f.endsWith('.gz'))
+      for (const name of gz) {
+        if (files.includes(`${name}.drops`)) continue
+        const m = /^\d{4}-(\d+)-(\d+)\.gz$/.exec(name)
+        if (!m) continue
+        const data = await readFile(join(rangeDir, name))
+        await Corpus.write(range, { from: Number(m[1]), to: Number(m[2]) }, data, Number.POSITIVE_INFINITY, filter)
+        before += data.length
+        after += (await readFile(join(rangeDir, name))).length
+        chunks++
+      }
+      // A complete range's range.json says it's filtered now (Corpus.write only writes it at the last chunk).
+      const json = join(rangeDir, 'range.json')
+      if (gz.length > 0 && (await access(json).then(() => true, () => false))) await writeFile(json, JSON.stringify({ ...range, filtered: FILTERED }))
+      if (chunks > 0 && chunks % 1000 < gz.length) this.logger.info(`${chunks} chunks filtered: ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
+    }
+    this.logger.info(`Done: ${chunks} chunks of ${ranges.length} ranges filtered, ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
+  }
+}

commands/edge_bench.ts

@@ -78,7 +78,17 @@ export default class EdgeBench extends BaseCommand {
     for (let from = this.start; from <= end; from += this.chunk, index++) {
       const to = Math.min(from + this.chunk - 1, end)
       const started = Date.now()
-      const raw = await client.xoverRaw(from, to)
+      let raw = await client.xoverRaw(from, to)
+      // Eweka cuts some ranges' compressed reply short (the same bytes every time): fetched plain then.
+      try {
+        unzipSync(raw)
+      } catch {
+        const plain = await XoverClient.connect({ host: env.get('NNTP_HOST')!, port: env.get('NNTP_PORT')!, user: env.get('NNTP_USER'), password: env.get('NNTP_PASSWORD'), secure: env.get('NNTP_SECURE', true), compress: false, timeoutMs: 300_000 })
+        await plain.group(this.group)
+        raw = await plain.xoverRaw(from, to)
+        plain.close()
+        this.logger.warning(`chunk ${index}: the compressed reply didn't inflate; fetched plain`)
+      }
       await writeFile(join(this.dir, `${String(index).padStart(4, '0')}-${from}-${to}.gz`), raw)
       this.logger.info(`chunk ${index}: ${from}–${to}, ${(raw.length / 1e6).toFixed(1)} MB in ${((Date.now() - started) / 1000).toFixed(1)} s`)
     }
@@ -98,7 +108,7 @@ export default class EdgeBench extends BaseCommand {
     let text = 0
     let peakRss = 0
     let peakDetail = ''
-    const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
+    const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, samples: 0 }
     const counts = new Map<string, number>()
     const keys = new Set<string>()
     let batches = 0
@@ -154,7 +164,6 @@ export default class EdgeBench extends BaseCommand {
         out.payload += r.payload.length
       }
       out.candidates += batch.candidates.length
-      out.fragments += batch.fragments.length
       out.samples += batch.samples.length
       for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles)
       const m = process.memoryUsage()
@@ -172,7 +181,7 @@ export default class EdgeBench extends BaseCommand {
     this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
     this.logger.info(`  inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`)
     this.logger.info(`  ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
-    this.logger.info(`  kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
+    this.logger.info(`  kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.samples} samples`)
     for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(`  dropped ${k}: ${v.toLocaleString('en')} articles`)
     if (this.store) this.logger.info(`  stored ${batches} batches in ${(storeMs / 1000).toFixed(1)} s (${(storeMs / Math.max(1, out.releases)).toFixed(2)} ms per release chunk)`)
   }

commands/edge_node.ts

@@ -8,6 +8,8 @@ import { gzip, unzip } from 'node:zlib'
 import { RangeIndexer, decodeFrames, isKeepalive, throttle, toWire, type EdgeBatch, type Lease } from '#lib/edge/index'
 
 const IDLE_MS = 30_000
+/** A batch is posted once this many files (completed, or cut at an edge) wait, whatever its articles. */
+const MAX_PENDING_FILES = 100_000
 const unzipAsync = promisify(unzip)
 const gzipAsync = promisify(gzip)
 const RETRIES = 8
@@ -166,7 +168,8 @@ export default class EdgeNode extends BaseCommand {
       indexer.addText(text.toString('utf8'), chunk.to + 1)
       reachedEnd = chunk.to >= range.end
       if (reachedEnd) indexer.finish()
-      if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles) {
+      // Also early when many files wait: a picture group's 500k articles made batches over the server's 256 MB limit (HTTP 413).
+      if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles || indexer.pending >= MAX_PENDING_FILES) {
         post(indexer.take())
         batchFrom = chunk.to + 1
       }

commands/nntp_probe.ts

@@ -0,0 +1,117 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import env from '#start/env'
+import { XoverClient } from '#lib/nntp/index'
+
+/**
+ * What the news server does with XOVER spans around an article that made a request hang (ranges:dates: one point
+ * per group timed out after 2 minutes, again on a fresh connection; LOG.md 2026-10-25 (105)). Each probe on a
+ * connection of its own with a short timeout: the span asked for, the reply, how many lines, how long. First the
+ * span itself and wider ones around it, then halves of it down to single articles, then a span past `high`.
+ */
+export default class NntpProbe extends BaseCommand {
+  static commandName = 'nntp:probe'
+  static description =
+    'Probes XOVER spans around an article that hangs, bisecting to the articles at fault'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'Group' })
+  declare group: string
+
+  @args.string({ description: 'First article of the span that hung' })
+  declare article: string
+
+  @flags.number({ description: 'Articles in the span (default 32)' })
+  declare size?: number
+
+  @flags.number({ description: 'Seconds without data before a probe is given up (default 20)' })
+  declare timeout?: number
+
+  @flags.boolean({ description: 'Only the span itself and the wider ones, no bisection' })
+  declare noBisect: boolean
+
+  @flags.boolean({ description: 'Without XFEATURE COMPRESS GZIP' })
+  declare plain: boolean
+
+  @flags.string({
+    description: 'Only spans from the article of these sizes, e.g. "8,16,24,32,48,64"',
+  })
+  declare sizes?: string
+
+  private timeoutMs = 20_000
+  private info = { count: 0, low: 0, high: 0 }
+
+  async run() {
+    this.timeoutMs = (this.timeout ?? 20) * 1000
+    const a = Number(this.article)
+    const size = this.size ?? 32
+    const client = await this.connect()
+    this.info = await client.group(this.group)
+    client.close()
+    this.logger.info(
+      `${this.group}: ${this.info.count} articles, ${this.info.low}–${this.info.high}${this.plain ? '; plain' : '; compressed'}.`
+    )
+    const span = (from: number, to: number) =>
+      [Math.max(this.info.low, from), Math.min(this.info.high, to)] as const
+    if (this.sizes) {
+      for (const n of this.sizes.split(',').map(Number)) await this.probe(...span(a, a + n - 1))
+      return
+    }
+    this.logger.info('The span and wider ones:')
+    await this.probe(...span(a, a + size - 1))
+    await this.probe(...span(a, a + size * 32 - 1))
+    await this.probe(...span(a - size * 32, a + size - 1))
+    await this.probe(...span(a - 50_000, a + 49_999))
+    this.logger.info('Past the end, and an empty span before it:')
+    await this.probe(this.info.high + 100, this.info.high + 131)
+    await this.probe(this.info.high + 1, this.info.high + 1)
+    if (this.noBisect) return
+    this.logger.info('Halves of the span:')
+    await this.bisect(a, Math.min(this.info.high, a + size - 1))
+  }
+
+  /** Halves of a span that hangs, down to single articles; a half that answers isn't split further. */
+  private async bisect(from: number, to: number) {
+    const hung = await this.probe(from, to)
+    if (!hung || from === to) return
+    const mid = Math.floor((from + to) / 2)
+    await this.bisect(from, mid)
+    await this.bisect(mid + 1, to)
+  }
+
+  /** One XOVER on a fresh connection; true when it timed out. */
+  private async probe(from: number, to: number): Promise<boolean> {
+    const startedAt = Date.now()
+    let client: XoverClient | null = null
+    try {
+      client = await this.connect()
+      await client.group(this.group)
+      const rows = await client.xover(from, to)
+      const t = client.lastTiming
+      this.logger.info(
+        `  XOVER ${from}-${to}: ${rows.length} lines in ${Date.now() - startedAt} ms (reply ${t.replyMs.toFixed(0)}, body ${t.bodyMs.toFixed(0)})${rows.length > 0 ? `; first ${rows[0][0]}, last ${rows[rows.length - 1][0]}` : ''}`
+      )
+      return false
+    } catch (error: any) {
+      const timedOut = /timed out/i.test(error.message)
+      this.logger.warning(
+        `  XOVER ${from}-${to}: ${error.message} after ${Date.now() - startedAt} ms${client ? `, ${client.buffered} bytes buffered: ${JSON.stringify(client.bufferedHead().toString('latin1'))} (${client.bufferedHead().toString('hex')})` : ''}`
+      )
+      return timedOut
+    } finally {
+      client?.close()
+    }
+  }
+
+  private connect() {
+    return XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: !this.plain,
+      timeoutMs: this.timeoutMs,
+    })
+  }
+}

commands/par2_prune.ts

@@ -0,0 +1,17 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { Par2Pruner } from '#services/par2_pruner'
+
+/** The par2 prune (Par2Pruner) in one go: a whole pass from the first set, instead of the merger's 2 minutes an hour. */
+export default class Par2Prune extends BaseCommand {
+  static commandName = 'par2:prune'
+  static description = 'Drops the par2 file lists the merge can no longer use: one whole pass over every set, then exits'
+  static options: CommandOptions = { startApp: true }
+
+  async run() {
+    await db.from('worker_cursors').where('name', 'par2-prune').update({ key: '' })
+    const round = await new Par2Pruner().round()
+    this.logger.info(`Pruned ${round.rows} par2 file rows: ${round.complete} complete sets, ${round.judged} judged, ${round.final} final.`)
+  }
+}

commands/pending_classes.ts

@@ -0,0 +1,105 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import coordinatorConfig from '#config/coordinator'
+import { describeClass } from '#lib/release/index'
+
+/**
+ * The pending ladder's classes (LOG.md (130)) on the rows from before them: `klass` (the SQL pending_class) and
+ * `target_bytes` (the first target's bytes, when its candidates carry them). The finished rows (named, exhausted,
+ * naming) in one scan, seeding each class's record, which gets its verdict (dropped) as the ladder gives it; the
+ * waiting rows in windows of created_at by their index (a scan per window starved the prober on the HDD); then
+ * pending_classes' `waiting` counted afresh from the claim's index. --report shows the record.
+ */
+export default class PendingClasses extends BaseCommand {
+  static commandName = 'pending:classes'
+  static description =
+    'Classes the pending releases from before the classes and recounts pending_classes; --report shows the record'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Hours of created_at per window' })
+  declare hours: number
+
+  @flags.boolean({ description: 'Show the classes with the best and worst records, write nothing' })
+  declare report: boolean
+
+  async run() {
+    if (this.report) return this.showReport()
+    const hours = this.hours ?? 6
+    const started = Date.now()
+    // The finished rows (named, exhausted, naming; 6.5M on 2026-11-02) in one pass: a scan of the table, no index
+    // reaches them by status; they seed each class's record.
+    const { rows: seeded } = await db.rawQuery(
+      `WITH classed AS (
+         UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
+           target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
+         WHERE klass = 0 AND status IN (1, 3, 4)
+         RETURNING tool, group_id, klass, status),
+       upserted AS (
+         INSERT INTO pending_classes (tool, group_id, klass, probed, named)
+         SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed GROUP BY 1, 2, 3
+         ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named
+         RETURNING 1)
+       SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM upserted) AS classes`
+    )
+    this.logger.info(
+      `Finished rows: ${seeded[0].classed} classed into ${seeded[0].classes} classes (${Math.round((Date.now() - started) / 1000)} s).`
+    )
+    const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
+    await db.rawQuery(
+      `UPDATE pending_classes SET dropped = true, swept_at = NULL WHERE NOT dropped AND probed >= ?::int AND named::float <= probed * ?::float`,
+      [pendingMinProbes, pendingMaxNamed]
+    )
+    // The waiting rows by windows of created_at (pending_releases_waiting_idx reaches them).
+    const { rows: span } = await db.rawQuery(
+      `SELECT (SELECT min(created_at) FROM pending_releases WHERE status = 0) AS lo, (SELECT max(created_at) FROM pending_releases WHERE status = 0) AS hi`
+    )
+    if (span[0].lo) {
+      let from = new Date(span[0].lo)
+      const hi = new Date(span[0].hi)
+      let classed = 0
+      while (from <= hi) {
+        const to = new Date(from.getTime() + hours * 3600_000)
+        const result = await db.rawQuery(
+          `UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
+             target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
+           WHERE status = 0 AND created_at >= ? AND created_at < ? AND klass = 0`,
+          [from.toISOString(), to.toISOString()]
+        )
+        classed += result.rowCount ?? 0
+        from = to
+        this.logger.info(
+          `Waiting rows up to ${to.toISOString().slice(0, 16)}: ${classed} classed (${Math.round((Date.now() - started) / 1000)} s).`
+        )
+      }
+    }
+    // The waiting counts from the rows themselves (the claim, the discard and the batch store keep them from here).
+    await db.rawQuery(`
+      INSERT INTO pending_classes (tool, group_id, klass, waiting)
+      SELECT tool, group_id, klass, count(*) FROM pending_releases WHERE status = 0 AND finished GROUP BY 1, 2, 3
+      ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = excluded.waiting`)
+    await db.rawQuery(
+      `UPDATE pending_classes c SET waiting = 0 WHERE waiting > 0 AND NOT EXISTS (
+         SELECT 1 FROM pending_releases p WHERE p.status = 0 AND p.finished AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass)`
+    )
+    const { rows: totals } = await db.rawQuery(
+      `SELECT count(*) AS classes, coalesce(sum(waiting), 0) AS waiting, count(*) FILTER (WHERE dropped) AS dropped FROM pending_classes`
+    )
+    this.logger.info(
+      `Done: ${totals[0].classes} classes, ${totals[0].waiting} waiting, ${totals[0].dropped} dropped.`
+    )
+  }
+
+  private async showReport() {
+    const { rows } = await db.rawQuery(
+      `SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named, c.waiting, c.dropped FROM pending_classes c JOIN groups g ON g.id = c.group_id
+       WHERE c.probed >= 20 ORDER BY c.named::float / c.probed DESC, c.probed DESC`
+    )
+    const line = (r: any) =>
+      `  ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} probed ${String(r.probed).padStart(7)} named ${String(Math.round((100 * r.named) / r.probed)).padStart(3)}% waiting ${String(r.waiting).padStart(8)}${r.dropped ? ' dropped' : ''}`
+    this.logger.info(`${rows.length} classes with 20+ climbs; the best:`)
+    for (const r of rows.slice(0, 25)) this.logger.info(line(r))
+    this.logger.info('The worst:')
+    for (const r of rows.slice(-25)) this.logger.info(line(r))
+  }
+}

commands/probe_run.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Prober } from '#services/prober'
+
+/** Phase 2's prober (REBUILD.md, part 6): reads the probe queue's articles and stores what they show. */
+export default class ProbeRun extends BaseCommand {
+  static commandName = 'probe:run'
+  static description = "Reads the probe queue's articles (PROBE_CONNECTIONS) and stores what they show"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  async run() {
+    const prober = new Prober()
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      this.logger.info('Stopping: finishing the articles in flight.')
+      prober.stop()
+      await finished
+    })
+    await prober.run()
+    this.logger.info('Prober stopped.')
+    done!()
+    await this.terminate()
+  }
+}

commands/ranges_capture.ts

@@ -0,0 +1,80 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+
+/** Bytes a filtered corpus article takes on disk (334 GB for 18.0B articles, 2026-10-25). */
+const BYTES_PER_ARTICLE = 18.5
+
+/**
+ * Marks every active group's ranges posted in a window (by ranges.est_date, from ranges:dates) `capture`, so the
+ * corpus fill fetches them: voyager 3's test data, a year of all groups at a time (REBUILD.md, corpus). Planned
+ * ranges not started get --priority (the fill's order, then the indexing's once there's room); done ones are
+ * fetched again into the corpus only. Ranges leased or half spooled are left out (their first chunks would be
+ * missing) and reported.
+ */
+export default class RangesCapture extends BaseCommand {
+  static commandName = 'ranges:capture'
+  static description =
+    'Marks the ranges posted in a window for the corpus: --from 2025-01-01 --to 2026-01-01'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.string({ description: 'First day of the window (ISO date)', required: true })
+  declare from: string
+
+  @flags.string({ description: 'Day after the window (ISO date)', required: true })
+  declare to: string
+
+  @flags.number({
+    description: 'Priority for the planned ranges marked (default 0: before the history)',
+  })
+  declare priority?: number
+
+  @flags.boolean({ description: 'Only report what the window holds' })
+  declare dryRun: boolean
+
+  async run() {
+    const from = new Date(this.from)
+    const to = new Date(this.to)
+    if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from)
+      return this.logger.error('--from and --to: ISO dates, from before to')
+    const priority = this.priority ?? 0
+    // A range covers est_date up to the next range's; the last range of a group up to now.
+    const hits = `
+      WITH spans AS (
+        SELECT r.id, r.status, r.capture, r.spooled_chunks, r.end_id - r.start_id + 1 AS n, r.est_date AS d0,
+          lead(r.est_date) OVER (PARTITION BY r.group_id ORDER BY r.start_id) AS d1
+        FROM ranges r JOIN groups g ON g.id = r.group_id WHERE g.active AND r.est_date IS NOT NULL
+      ), hit AS (SELECT * FROM spans WHERE d0 < ? AND coalesce(d1, now()) > ?)`
+    const { rows } = await db.rawQuery(
+      `${hits}
+       SELECT CASE WHEN capture THEN 'marked already' WHEN status = 'planned' AND spooled_chunks = 0 THEN 'planned'
+                   WHEN status = 'done' THEN 'done' ELSE 'in flight' END AS kind,
+         count(*) AS ranges, round(sum(n) / 1e9, 2) AS articles
+       FROM hit GROUP BY 1 ORDER BY 1`,
+      [to, from]
+    )
+    const fetch = rows
+      .filter((r: any) => r.kind === 'planned' || r.kind === 'done')
+      .reduce((sum: number, r: any) => sum + Number(r.articles), 0)
+    this.logger.info(`${this.from} to ${this.to}:`)
+    for (const r of rows)
+      this.logger.info(`  ${r.kind}: ${r.ranges} ranges, ${r.articles}B articles`)
+    this.logger.info(
+      `To fetch: ${fetch.toFixed(2)}B articles, ~${((fetch * 1e9 * BYTES_PER_ARTICLE) / 1e12).toFixed(2)} TB in the corpus.`
+    )
+    if (this.dryRun) return
+    const { rows: marked } = await db.rawQuery(
+      `${hits}
+       UPDATE ranges r SET capture = true,
+         priority = CASE WHEN r.status = 'planned' THEN ? ELSE r.priority END
+       FROM hit WHERE r.id = hit.id AND NOT hit.capture
+         AND (hit.status = 'done' OR (hit.status = 'planned' AND hit.spooled_chunks = 0))
+       RETURNING r.status`,
+      [to, from, priority]
+    )
+    const planned = marked.filter((r: any) => r.status === 'planned').length
+    this.logger.info(
+      `Marked ${planned} planned ranges at priority ${priority} and ${marked.length - planned} done ranges.`
+    )
+  }
+}

commands/ranges_dates.ts

@@ -0,0 +1,216 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { NntpShare } from '#services/nntp_shares'
+import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
+
+/**
+ * When each active group's articles were posted (group_dates, sampled from the news server) and from that every
+ * range's estimated posting date (ranges.est_date), so ranges can be chosen by era: ranges:capture marks a
+ * posting-time window of every group for the corpus (REBUILD.md, corpus). A group gets a sample every --every
+ * articles (16 … 1000): boneless (161B articles) ~320, three weeks apart; ~15k XOVER commands of 32 articles in all.
+ */
+export default class RangesDates extends BaseCommand {
+  static commandName = 'ranges:dates'
+  static description =
+    "Samples when each active group's articles were posted and estimates every range's date"
+  static options: CommandOptions = { startApp: true }
+
+  @flags.string({ description: 'Only groups whose name matches this SQL LIKE pattern' })
+  declare group?: string
+
+  @flags.number({ description: 'Articles between two samples of a group (default 500M)' })
+  declare every?: number
+
+  @flags.number({
+    description:
+      'Connections, a share of the account beside the coordinator and the prober (default 6)',
+  })
+  declare connections?: number
+
+  @flags.boolean({ description: 'Only groups without samples yet' })
+  declare missing: boolean
+
+  @flags.boolean({
+    description: 'No sampling: estimate the ranges from the samples stored already',
+  })
+  declare estimateOnly: boolean
+
+  async run() {
+    if (!this.estimateOnly) await this.sample()
+    await this.estimate()
+    await this.report()
+  }
+
+  private groups(missing = this.missing) {
+    const q = db.from('groups').where('active', true).select('id', 'name').orderBy('priority')
+    if (this.group) q.where('name', 'like', this.group)
+    if (missing)
+      q.whereNotExists((sub) =>
+        sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
+      )
+    return q
+  }
+
+  private async sample() {
+    const every = this.every ?? 500_000_000
+    const connections = this.connections ?? 6
+    const groups = await this.groups()
+    // Connections of its own, without compression: Eweka cuts a compressed reply at the first ".\r\n" in its
+    // output, which 16 of 14k small XOVERs hit (TruncatedReply; LOG.md 2026-10-25 (105)); plain, every one
+    // answers, and 32 lines are a few kB. Taken from the account as a share (the coordinator and the prober make
+    // room for them), as many as asked for.
+    const workers = Math.max(1, Math.min(connections, groups.length))
+    const share = new NntpShare(
+      'dates',
+      coordinatorConfig.nntpShares.coordinatorPriority + 1,
+      workers,
+      workers
+    )
+    let open = 0
+    const sync = () =>
+      share
+        .sync(workers, open)
+        .catch((error) => this.logger.warning(`NNTP share: ${error.message}`))
+    await sync()
+    const sharing = setInterval(sync, coordinatorConfig.nntpShares.everyMs)
+    this.logger.info(
+      `Sampling ${groups.length} groups over ${workers} plain connections, a sample every ${every / 1e6}M articles.`
+    )
+    const startedAt = Date.now()
+    let done = 0
+    let commands = 0
+    let samples = 0
+    const queue = [...groups]
+    const worker = async () => {
+      let client: XoverClient | null = null
+      const use = async <T>(work: (c: XoverClient) => Promise<T>): Promise<T> => {
+        if (!client) {
+          client = await XoverClient.connect({
+            host: env.get('NNTP_HOST')!,
+            port: env.get('NNTP_PORT', 563),
+            user: env.get('NNTP_USER'),
+            password: env.get('NNTP_PASSWORD'),
+            secure: env.get('NNTP_SECURE', true),
+            compress: false,
+            timeoutMs: coordinatorConfig.nntpTimeoutMs,
+          })
+          open++
+        }
+        try {
+          return await work(client)
+        } catch (error) {
+          client.close()
+          client = null
+          open--
+          throw error
+        }
+      }
+      for (let g = queue.shift(); g; g = queue.shift()) {
+        try {
+          const result = await sampleGroup(use, g.name, { every, size: 32 })
+          for (const f of result.failed) this.logger.warning(`${g.name} at ${f}`)
+          await db.transaction(async (trx) => {
+            await trx.from('group_dates').where('group_id', g.id).delete()
+            if (result.samples.length > 0)
+              await trx.table('group_dates').multiInsert(
+                result.samples.map((s) => ({
+                  group_id: g.id,
+                  article: s.article,
+                  posted_at: s.postedAt,
+                }))
+              )
+          })
+          commands += result.commands
+          samples += result.samples.length
+          if (result.samples.length < 2)
+            this.logger.warning(
+              `${g.name}: ${result.samples.length} samples over ${result.low}–${result.high}.`
+            )
+        } catch (error: any) {
+          this.logger.warning(`${g.name}: ${error.message}`)
+        }
+        done++
+        if (done % 50 === 0)
+          this.logger.info(
+            `${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
+          )
+      }
+      if (client) {
+        ;(client as XoverClient).close()
+        open--
+      }
+    }
+    try {
+      await Promise.all(Array.from({ length: workers }, worker))
+    } finally {
+      clearInterval(sharing)
+      await share.sync(0, 0).catch(() => {})
+    }
+    this.logger.info(
+      `Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
+    )
+  }
+
+  /** Every range of the groups sampled gets est_date from the group's curve (1000 a statement). */
+  private async estimate() {
+    const groups = await this.groups(false).whereExists((sub) =>
+      sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
+    )
+    let updated = 0
+    for (const g of groups) {
+      const rows = await db
+        .from('group_dates')
+        .where('group_id', g.id)
+        .orderBy('article')
+        .select('article', 'posted_at')
+      const samples: DateSample[] = rows.map((r) => ({
+        article: Number(r.article),
+        postedAt: new Date(r.posted_at),
+      }))
+      const ranges = await db
+        .from('ranges')
+        .where('group_id', g.id)
+        .orderBy('start_id')
+        .select('id', 'start_id')
+      for (let i = 0; i < ranges.length; i += 1000) {
+        const batch = ranges.slice(i, i + 1000)
+        const values = batch.map(() => '(?::bigint, ?::timestamptz)').join(', ')
+        const bindings = batch.flatMap((r) => [
+          Number(r.id),
+          estimateDate(samples, Number(r.start_id))!.toISOString(),
+        ])
+        await db.rawQuery(
+          `UPDATE ranges AS r SET est_date = v.d FROM (VALUES ${values}) AS v(id, d) WHERE r.id = v.id`,
+          bindings
+        )
+        updated += batch.length
+      }
+    }
+    this.logger.info(`${updated} ranges of ${groups.length} groups dated.`)
+  }
+
+  /** The planned history by posting year: what a window would take. */
+  private async report() {
+    const { rows } = await db.rawQuery(`
+      SELECT extract(year FROM est_date)::int AS year, count(*) AS ranges, count(DISTINCT group_id) AS groups,
+        round(sum(end_id - start_id + 1) / 1e9, 1) AS articles,
+        round(sum(end_id - start_id + 1) FILTER (WHERE status = 'done') / 1e9, 1) AS done,
+        round(sum(end_id - start_id + 1) FILTER (WHERE capture) / 1e9, 1) AS capture
+      FROM ranges WHERE est_date IS NOT NULL GROUP BY 1 ORDER BY 1`)
+    const undatedRows = await db.rawQuery(
+      `SELECT count(*) AS undated FROM ranges WHERE est_date IS NULL`
+    )
+    const undated = undatedRows.rows[0].undated
+    this.logger.info(
+      `By estimated posting year (B articles; done: indexed, capture: marked for the corpus); ${undated} ranges undated:`
+    )
+    for (const r of rows)
+      this.logger.info(
+        `  ${r.year}: ${String(r.ranges).padStart(7)} ranges of ${String(r.groups).padStart(3)} groups, ${String(r.articles).padStart(6)}B articles, ${r.done}B done, ${r.capture}B capture`
+      )
+  }
+}

commands/ranges_merge_tail.ts

@@ -0,0 +1,23 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Planner from '#services/planner'
+
+/**
+ * Merges adjacent planned tail ranges (Planner.mergeTail): the tail planner added a sliver a group every round
+ * while nothing leased the tail; now it grows the group's open range instead, and this packs what is there.
+ */
+export default class RangesMergeTail extends BaseCommand {
+  static commandName = 'ranges:merge-tail'
+  static description = 'Merges adjacent planned tail ranges into ranges of RANGE_ARTICLES'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'Only count what would be merged' })
+  declare dryRun: boolean
+
+  async run() {
+    const { candidates, merged, dropped } = await Planner.mergeTail(this.dryRun)
+    this.logger.info(
+      `${candidates} planned tail ranges: ${dropped} ${this.dryRun ? 'would merge' : 'merged'} into ${merged} grown ranges (${candidates - dropped} left).`
+    )
+  }
+}

commands/ranges_plan.ts

@@ -1,18 +1,26 @@
-import { BaseCommand } from '@adonisjs/core/ace'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
 import type { CommandOptions } from '@adonisjs/core/types/ace'
 import Planner from '#services/planner'
 import NntpPool from '#services/nntp_pool'
 
-/** Cuts every active group's history (and new articles since) into ranges. */
+/** Cuts new groups' history into ranges, or (--tail) plans one round of the live tail. */
 export default class RangesPlan extends BaseCommand {
   static commandName = 'ranges:plan'
-  static description = "Plans ranges for the indexed groups' articles"
+  static description = "Plans ranges for new groups' history, or one round of the live tail"
   static options: CommandOptions = { startApp: true }
 
+  @flags.boolean({ description: 'Plan the live tail once (what coordinator:run does every few minutes)' })
+  declare tail: boolean
+
   async run() {
     try {
-      const { groups, ranges } = await Planner.planAll()
-      this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
+      if (this.tail) {
+        const { groups, ranges, extended } = await Planner.planTail()
+        this.logger.info(`Tail: ${ranges} ranges added, ${extended} grown, over ${groups} groups.`)
+      } else {
+        const { groups, ranges } = await Planner.planAll()
+        this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
+      }
     } finally {
       await NntpPool.close()
     }

commands/releases_duplicates.ts

@@ -0,0 +1,82 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+
+/**
+ * How often a release holds a file name more than once (decoded from its
+ * chunks): copies of one upload posted again (multi-posting, reposts that
+ * share a key) or different files that share a name. Copies of the same
+ * size are counted as the same file posted again.
+ */
+export default class ReleasesDuplicates extends BaseCommand {
+  static commandName = 'releases:duplicates'
+  static description = 'Counts file names a release holds more than once, in a sample'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
+  declare sample: number
+
+  async run() {
+    const ids: string[] = (
+      await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])
+    ).rows.map((r: { id: string }) => r.id)
+    const kind = (name: string) =>
+      /\.(jpe?g|png|gif|bmp|webp)$/i.test(name) ? 'image'
+      : /\.(mp3|flac|m4a|ogg|wav)\b/i.test(name) ? 'audio'
+      : /\.(par2)$/i.test(name) ? 'par2'
+      : /\.(rar|r\d\d|7z|zip|\d{3})$/i.test(name) ? 'archive'
+      : /\.(pdf|epub|mobi|cbr|cbz|djvu|chm)$/i.test(name) ? 'document'
+      : /\.(nfo|sfv|nzb|txt|diz)$/i.test(name) ? 'sidecar'
+      : 'other'
+    let releases = 0
+    let withDuplicates = 0
+    let files = 0
+    let extra = 0
+    let completeOnlyByCopies = 0
+    const byKind = new Map<string, number>()
+    const how = { sameFile: 0, differentFile: 0, sameDay: 0, laterRepost: 0 }
+    const examples: string[] = []
+    for (const id of ids) {
+      const release = await db.from('releases').where('id', id).select('name', 'file_total', 'files').first()
+      const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+      const byName = new Map<string, Map<string, { parts: number; bytes: number; at: number | null }>>()
+      for (const c of chunks) {
+        for (const f of decodeChunk(c.payload)) {
+          const copies = byName.get(f.name) ?? new Map()
+          copies.set(f.segments[0]?.[1] ?? '', { parts: f.segments.length, bytes: f.segments.reduce((n, [, , b]) => n + b, 0), at: f.postedAt })
+          byName.set(f.name, copies)
+        }
+      }
+      releases++
+      let dup = false
+      for (const [name, copies] of byName) {
+        files += copies.size
+        if (copies.size < 2) continue
+        dup = true
+        extra += copies.size - 1
+        byKind.set(kind(name), (byKind.get(kind(name)) ?? 0) + copies.size - 1)
+        const list = [...copies.values()]
+        // The same file again: as many parts, within 3% in bytes (article sizes include headers).
+        const same = list.every((c) => c.parts === list[0].parts && Math.abs(c.bytes - list[0].bytes) <= 0.03 * list[0].bytes)
+        if (same) how.sameFile++
+        else how.differentFile++
+        const times = list.map((c) => c.at).filter((t): t is number => t !== null)
+        if (times.length > 1 && Math.max(...times) - Math.min(...times) > 86_400_000) how.laterRepost++
+        else how.sameDay++
+        if (!same && examples.length < 8) examples.push(`${id} ${release.name.slice(0, 60)} | ${name.slice(0, 60)} | ${list.map((c) => `${c.parts}p ${(c.bytes / 1e6).toFixed(1)}MB`).join(', ')}`)
+      }
+      if (!dup) continue
+      withDuplicates++
+      const total = Number(release.file_total ?? 0)
+      if (total && Number(release.files) >= total && byName.size < total) completeOnlyByCopies++
+    }
+    const pct = (n: number, of: number) => `${((100 * n) / Math.max(1, of)).toFixed(2)}%`
+    this.logger.info(`${releases} releases with 2+ files sampled`)
+    this.logger.info(`  with a file name more than once: ${withDuplicates} (${pct(withDuplicates, releases)})`)
+    this.logger.info(`  extra copies: ${extra} of ${files} files (${pct(extra, files)}); by type: ${[...byKind].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${k} ${n}`).join(', ')}`)
+    this.logger.info(`  duplicated names: same file again ${how.sameFile}, different (parts or size) ${how.differentFile}; copies the same day ${how.sameDay}, a day or more apart ${how.laterRepost}`)
+    this.logger.info(`  'complete' only by counting copies: ${completeOnlyByCopies}`)
+    for (const e of examples) this.logger.info(`  different: ${e}`)
+  }
+}

commands/releases_judge.ts

@@ -0,0 +1,38 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { ReleaseJudge, RULES } from '#services/release_judge'
+
+/** The junk judge (REBUILD.md, junk): releases judged after their probes; verdicts hide them from search. */
+export default class ReleasesJudge extends BaseCommand {
+  static commandName = 'releases:judge'
+  static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable): the changed ones every few minutes, all daily'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One full round, then exit' })
+  declare once: boolean
+
+  @flags.boolean({ description: 'With --once: count only, write no verdicts' })
+  declare dry: boolean
+
+  async run() {
+    const judge = new ReleaseJudge()
+    if (this.once) {
+      const round = await judge.full(this.dry)
+      for (const rule of RULES) {
+        const r = round.rules[rule.name]
+        if (r) this.logger.info(`${rule.name}: ${r.matched} (+${r.added} −${r.removed}) — ${rule.says}`)
+      }
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      judge.stop()
+      await finished
+    })
+    await judge.run()
+    this.logger.info('Judge stopped.')
+    done!()
+  }
+}

commands/releases_merge.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Par2Merger } from '#services/par2_merger'
+
+/** The par2 merge (REBUILD.md, phase 2): releases whose files one par2 set describes become one. */
+export default class ReleasesMerge extends BaseCommand {
+  static commandName = 'releases:merge'
+  static description = 'Merges the releases whose files one par2 set describes (what the prober read), every minute'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round over every set, then exit' })
+  declare once: boolean
+
+  async run() {
+    const merger = new Par2Merger()
+    if (this.once) {
+      const round = await merger.round()
+      this.logger.info(`${round.sets} sets checked: ${round.releases} releases merged into ${round.merges}.`)
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      merger.stop()
+      await finished
+    })
+    await merger.run()
+    this.logger.info('Merger stopped.')
+    done!()
+    await this.terminate()
+  }
+}

commands/releases_merge_names.ts

@@ -0,0 +1,65 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { Par2Merger } from '#services/par2_merger'
+
+/**
+ * The par2 merger's name pass over sets found before it was built (2026-10-23): every set with its file list, newest
+ * first, by set row. --dry-run counts what it would merge and shows examples, merging nothing.
+ */
+export default class ReleasesMergeNames extends BaseCommand {
+  static commandName = 'releases:merge-names'
+  static description = "Merges releases by their par2 set's file names, for sets found before the name pass; --dry-run shows only"
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Sets to look at (newest first); all without' })
+  declare sets: number
+
+  @flags.boolean({ description: 'Count and show, merge nothing' })
+  declare dryRun: boolean
+
+  async run() {
+    const merger = new Par2Merger()
+    // Keyset on (newest created_at, set_id): sets stored in one batch share created_at, and the timestamp stays text
+    // so microseconds survive the round trip.
+    let before = ['9999-12-31', '']
+    let looked = 0
+    let merges = 0
+    let releases = 0
+    const examples: string[] = []
+    while (this.sets === undefined || looked < this.sets) {
+      const { rows } = await db.rawQuery(
+        `SELECT set_id, at::text AS at FROM (
+           SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
+           GROUP BY s.set_id) x
+         WHERE (at, set_id) < (?::timestamptz, ?) ORDER BY at DESC, set_id DESC LIMIT ${this.dryRun ? 50 : 500}`,
+        before
+      )
+      if (rows.length === 0) break
+      for (const { set_id: setId } of rows) {
+        looked++
+        if (this.dryRun) {
+          const found = await db.transaction(async (trx) => merger.nameMatches(trx, setId.trim()))
+          if (found) {
+            merges++
+            releases += found.size
+            if (examples.length < 15) {
+              const { rows: names } = await db.rawQuery(`SELECT id, name, files FROM releases WHERE id = ANY(?) ORDER BY id`, [[...found.keys()]])
+              examples.push(names.map((n: any) => `${n.name.slice(0, 50)} [${n.files}]`).join(' + '))
+            }
+          }
+        } else {
+          const merged = await merger.mergeByName(setId.trim())
+          if (merged > 0) {
+            merges++
+            releases += merged + 1
+          }
+        }
+      }
+      before = [rows.at(-1).at, rows.at(-1).set_id]
+      if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
+    }
+    this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
+    for (const e of examples) this.logger.info(`  ${e.slice(0, 300)}`)
+  }
+}

commands/releases_par2_complete.ts

@@ -0,0 +1,38 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decidePar2Completeness } from '#services/par2_completeness'
+
+/** Releases per batch. */
+const BATCH = 1000
+
+/**
+ * Decides completeness by par2 set (app/services/par2_completeness.ts) for the releases that had their par2 read
+ * before it was built (2026-10-23): every holder of a set whose file list is still there, by release id. Once.
+ */
+export default class ReleasesPar2Complete extends BaseCommand {
+  static commandName = 'releases:par2-complete'
+  static description = 'Decides completeness by their par2 sets for settled releases without a counter, then exits'
+  static options: CommandOptions = { startApp: true }
+
+  async run() {
+    let after = 0
+    let looked = 0
+    let decided = 0
+    for (;;) {
+      const { rows } = await db.rawQuery(
+        `SELECT DISTINCT s.release_id FROM par2_sets s
+         WHERE s.release_id > ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
+         ORDER BY s.release_id LIMIT ?`,
+        [after, BATCH]
+      )
+      if (rows.length === 0) break
+      const ids = rows.map((r: { release_id: string }) => Number(r.release_id))
+      decided += await decidePar2Completeness(db, ids)
+      looked += ids.length
+      after = ids.at(-1)!
+      if (looked % 50_000 < BATCH) this.logger.info(`${looked} releases with a par2 set looked at, ${decided} decided (up to id ${after}).`)
+    }
+    this.logger.info(`Done: ${looked} releases with a par2 set looked at, ${decided} decided.`)
+  }
+}

commands/releases_readable.ts

@@ -0,0 +1,90 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { isUnreadableName } from '#lib/release/index'
+
+/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
+const CURSOR = 'readable'
+
+/**
+ * The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
+ * readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
+ * false, so the readable filter hides them (an inner name from the prober still shows them), and their probe
+ * candidates marked unreadable, the prober's first tiers. By the release name,
+ * which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
+ * Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
+ */
+export default class ReleasesReadable extends BaseCommand {
+  static commandName = 'releases:readable'
+  static description =
+    'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Release ids per batch' })
+  declare batch: number
+
+  @flags.number({ description: 'Stop after this id (a sample)' })
+  declare toId: number
+
+  @flags.boolean({ description: 'Write the flag; without it, count and show a sample' })
+  declare apply: boolean
+
+  @flags.boolean({ description: 'Start from the first id, not the cursor' })
+  declare restart: boolean
+
+  async run() {
+    const batch = this.batch ?? 20_000
+    const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
+    const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
+    const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [
+      CURSOR,
+    ])
+    let after =
+      this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
+    let looked = 0
+    let changed = 0
+    const sample: string[] = []
+    const started = Date.now()
+    while (after < hi) {
+      const to = Math.min(after + batch, hi)
+      const { rows } = await db.rawQuery(
+        `SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`,
+        [after, to]
+      )
+      looked += rows.length
+      const ids: number[] = []
+      for (const r of rows) {
+        if (!isUnreadableName(r.name)) continue
+        ids.push(Number(r.id))
+        if (sample.length < 40) sample.push(r.name)
+      }
+      changed += ids.length
+      if (this.apply) {
+        if (ids.length > 0) {
+          await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
+            ids,
+          ])
+          // Their probe candidates to the unreadable tiers, which the prober takes first (LOG.md (130)).
+          await db.rawQuery(
+            `UPDATE probe_candidates SET unreadable = true WHERE release_id = ANY(?::bigint[]) AND NOT unreadable`,
+            [ids]
+          )
+        }
+        await db.rawQuery(
+          `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
+          [CURSOR, String(to)]
+        )
+      }
+      after = to
+      if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) {
+        this.logger.info(
+          `Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).`
+        )
+      }
+    }
+    this.logger.info(
+      `${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.`
+    )
+    if (!this.apply) for (const n of sample) this.logger.info(`  ${n}`)
+  }
+}

commands/releases_recategorize.ts

@@ -0,0 +1,92 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { DiskSpace } from '#services/disk_space'
+
+/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
+const CURSOR = 'recategorize'
+
+/**
+ * The categories of releases from before migration 5200 (release_category_for: the fixed name rules, the inner
+ * name, the groups; then the spots'), in batches by id: only rows whose category changes are written, each a new row
+ * version in a 70 GB table on the NVMe, so it waits while the filesystem is under --floor-gb free (autovacuum makes
+ * the dead versions reusable) and goes on from its cursor when run again.
+ */
+export default class ReleasesRecategorize extends BaseCommand {
+  static commandName = 'releases:recategorize'
+  static description = 'Recomputes release categories (name, inner name, groups, spots) in batches; --dry-run counts only'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Release ids per batch' })
+  declare batch: number
+
+  @flags.number({ description: 'Stop after this id (a sample with --dry-run)' })
+  declare toId: number
+
+  @flags.number({ description: 'Wait while the NVMe has less free (GB)' })
+  declare floorGb: number
+
+  @flags.boolean({ description: 'Count what would change, write nothing' })
+  declare dryRun: boolean
+
+  @flags.boolean({ description: 'Start from the first id, not the cursor' })
+  declare restart: boolean
+
+  async run() {
+    const batch = this.batch ?? 20_000
+    const space = new DiskSpace('recategorizing', '/', this.floorGb ?? 14)
+    const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
+    const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
+    const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
+    let after = this.restart || this.dryRun || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
+    const changes = new Map<string, number>()
+    let changed = 0
+    let looked = 0
+    const started = Date.now()
+    while (after < hi) {
+      while (!this.dryRun && (await space.short())) await sleep(60_000)
+      const to = Math.min(after + batch, hi)
+      const fresh = `release_category_for(name, inner_name, groups, NULL)`
+      if (this.dryRun) {
+        const { rows } = await db.rawQuery(
+          `SELECT category AS was, ${fresh} AS now, count(*)::int AS n FROM releases WHERE id > ? AND id <= ? GROUP BY 1, 2`,
+          [after, to]
+        )
+        for (const r of rows) {
+          looked += r.n
+          if (r.was === r.now) continue
+          changed += r.n
+          changes.set(`${r.was} → ${r.now}`, (changes.get(`${r.was} → ${r.now}`) ?? 0) + r.n)
+        }
+      } else {
+        const result = await db.rawQuery(
+          `UPDATE releases r SET category = x.c FROM (SELECT id, ${fresh} AS c FROM releases WHERE id > ? AND id <= ?) x
+           WHERE r.id = x.id AND r.category IS DISTINCT FROM x.c`,
+          [after, to]
+        )
+        changed += result.rowCount ?? 0
+        await db.rawQuery(
+          `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
+          [CURSOR, String(to)]
+        )
+      }
+      after = to
+      if (Math.floor(after / 5_000_000) !== Math.floor((after - batch) / 5_000_000)) {
+        this.logger.info(`Up to id ${after} of ${hi}: ${changed} changed${this.dryRun ? ` of ${looked}` : ''} (${Math.round((Date.now() - started) / 1000)} s).`)
+      }
+    }
+    if (this.dryRun) {
+      this.logger.info(`Would change ${changed} of ${looked}:`)
+      for (const [change, n] of [...changes].sort((a, b) => b[1] - a[1]).slice(0, 30)) this.logger.info(`  ${change}: ${n}`)
+      return
+    }
+    // The spots' categories for the linked releases whose names say nothing.
+    const spots = await db.rawQuery(
+      `UPDATE releases r SET category = spot_category(s.category, s.subcats)
+       FROM release_spots rs JOIN spots s ON s.message_id = rs.spot_id
+       WHERE r.id = rs.release_id AND r.category = 8010 AND spot_category(s.category, s.subcats) IS NOT NULL`
+    )
+    this.logger.info(`Done: ${changed} releases recategorized, ${spots.rowCount ?? 0} by their spots.`)
+  }
+}

commands/releases_regroup.ts

@@ -0,0 +1,87 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+import { releaseIdentity } from '#lib/release/index'
+import { sourceCode } from '#lib/release/source'
+
+/**
+ * What lib/release's identity makes of releases already in the catalog: a
+ * sample of releases (by source), their files decoded from the chunks and
+ * each given its release key again. Shows how many would now split, and
+ * into what: for checking an identity change on real data.
+ */
+export default class ReleasesRegroup extends BaseCommand {
+  static commandName = 'releases:regroup'
+  static description = 'Shows how the current release identity would regroup a sample of the catalog'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled per source', default: 2000 })
+  declare sample: number
+
+  @flags.array({ description: 'Release ids to show in full' })
+  declare show?: string[]
+
+  @flags.array({ description: 'Groups whose indexed releases are re-keyed together (merges show too)' })
+  declare group?: string[]
+
+  async run() {
+    for (const name of this.group ?? []) {
+      const { rows } = await db.rawQuery(
+        `SELECT DISTINCT c.release_id FROM release_chunks c JOIN blocks b ON b.id = c.block_id JOIN groups g ON g.id = b.group_id WHERE g.name = ?`,
+        [name]
+      )
+      const ids = rows.map((r: { release_id: string }) => r.release_id)
+      const all = new Map<string, { name: string; source: string; files: number; subject?: string }>()
+      for (const id of ids) {
+        for (const [key, g] of await this.regroup(Number(id))) {
+          const known = all.get(key)
+          if (known) known.files += g.files
+          else all.set(key, { ...g })
+        }
+      }
+      const sizes = [...all.values()].map((g) => g.files)
+      const single = sizes.filter((n) => n === 1).length
+      this.logger.info(`${name}: ${ids.length} releases now, ${all.size} with the current rule (${single} of one file)`)
+      for (const g of [...all.values()].sort((a, b) => b.files - a.files).slice(0, 6)) this.logger.info(`  ${g.files} files: ${g.name.slice(0, 120)} (${g.source})`)
+      const singles = [...all.values()].filter((g) => g.files === 1)
+      for (let i = 0; i < Math.min(8, singles.length); i++) this.logger.info(`  single: ${singles[Math.floor((i * singles.length) / 8)].subject?.slice(0, 150)}`)
+    }
+    if (this.group?.length) return
+    for (const id of this.show ?? []) {
+      const groups = await this.regroup(Number(id))
+      this.logger.info(`${id}: ${groups.size} release(s) now`)
+      for (const [, g] of groups) this.logger.info(`  ${g.files} files: ${g.name} (${g.source})`)
+    }
+    if (this.show?.length) return
+    for (const source of ['subject', 'post', 'stem']) {
+      const rows = await db.from('releases').where('source', sourceCode(source)).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
+      let split = 0
+      let pieces = 0
+      const examples: string[] = []
+      for (const r of rows) {
+        const groups = await this.regroup(Number(r.id))
+        if (groups.size <= 1) continue
+        split++
+        pieces += groups.size
+        if (examples.length < 5) examples.push(`${r.id} ${r.name} (${r.files} files) → ${groups.size}: ${[...groups.values()].slice(0, 3).map((g) => `${g.name} [${g.files}]`).join(' | ')}`)
+      }
+      this.logger.info(`${source}: ${rows.length} sampled, ${split} would split (into ${pieces} releases)`)
+      for (const e of examples) this.logger.info(`  ${e.slice(0, 260)}`)
+    }
+  }
+
+  private async regroup(id: number) {
+    const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+    const groups = new Map<string, { name: string; source: string; files: number; subject?: string }>()
+    for (const c of chunks) {
+      for (const f of decodeChunk(c.payload)) {
+        const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
+        const g = groups.get(r.key)
+        if (g) g.files++
+        else groups.set(r.key, { name: r.name, source: r.source, files: 1, subject: f.subject })
+      }
+    }
+    return groups
+  }
+}

commands/releases_upload_gaps.ts

@@ -0,0 +1,58 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+
+/**
+ * For telling uploads apart (re-uploads under one release key): how long
+ * one upload takes to post (the span of a release's files' dates, in
+ * releases without a file name twice), and how far apart the copies of a
+ * file name are in releases that have one twice.
+ */
+export default class ReleasesUploadGaps extends BaseCommand {
+  static commandName = 'releases:upload-gaps'
+  static description = 'Posting spans of uploads and gaps between re-uploads, in a sample'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
+  declare sample: number
+
+  async run() {
+    const ids: string[] = (await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])).rows.map((r: { id: string }) => r.id)
+    const bins = [10, 60, 180, 360, 720, 1440, 4320, Number.POSITIVE_INFINITY]
+    const label = (m: number) => (m === Number.POSITIVE_INFINITY ? '>3d' : m >= 1440 ? `≤${m / 1440}d` : m >= 60 ? `≤${m / 60}h` : `≤${m}min`)
+    const spans = new Array(bins.length).fill(0)
+    const gaps = new Array(bins.length).fill(0)
+    const bin = (minutes: number) => bins.findIndex((b) => minutes <= b)
+    for (const id of ids) {
+      const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+      const byName = new Map<string, number[]>()
+      const times: number[] = []
+      for (const c of chunks) {
+        for (const f of decodeChunk(c.payload)) {
+          if (f.postedAt === null) continue
+          times.push(f.postedAt)
+          byName.set(f.name, [...(byName.get(f.name) ?? []), f.postedAt])
+        }
+      }
+      if (times.length < 2) continue
+      const twice = [...byName.values()].filter((t) => t.length > 1)
+      if (twice.length === 0) {
+        spans[bin((Math.max(...times) - Math.min(...times)) / 60_000)]++
+        continue
+      }
+      // The gap between a name's copies (sorted; each step), so a release reposted 3 times counts 2 gaps per name.
+      for (const t of twice) {
+        t.sort((a, b) => a - b)
+        for (let i = 1; i < t.length; i++) gaps[bin((t[i] - t[i - 1]) / 60_000)]++
+      }
+    }
+    const show = (counts: number[]) => {
+      const total = counts.reduce((a, b) => a + b, 0)
+      let cumulative = 0
+      return bins.map((b, i) => `${label(b)} ${((100 * (cumulative += counts[i])) / Math.max(1, total)).toFixed(1)}%`).join(', ') + ` (n=${total})`
+    }
+    this.logger.info(`one upload's posting span (cumulative): ${show(spans)}`)
+    this.logger.info(`gap between copies of a name (cumulative): ${show(gaps)}`)
+  }
+}

commands/replay_corpus.ts

@@ -0,0 +1,396 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { spawn } from 'node:child_process'
+import { access, readdir, readFile, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import db from '@adonisjs/lucid/services/db'
+import coordinatorConfig from '#config/coordinator'
+import BatchStore from '#services/batch_store'
+import Leases from '#services/leases'
+import RuleSets from '#services/rule_sets'
+import { fromWire, type WireBatch } from '#lib/edge/index'
+
+/**
+ * Only a database named like one of these is written: the replay leases and finishes ranges and fills the catalog.
+ * Not the test database: the rows left behind failed 27 functional tests (LOG.md 2026-10-26 (111)). With
+ * REPLAY_INTO_CATALOG=1 any database but the test one: voyager 3 deployed, history replayed from the corpus into
+ * the catalog itself (deploy/replay-window.sh, LOG.md (116)).
+ */
+const SCRATCH_DB = /_(replay|scratch)$/
+const TEST_DB = /_test$/
+const NODE_NAME = 'replay'
+
+interface StoredRange {
+  id: number
+  start: number
+  end: number
+}
+
+/**
+ * The replay harness (REBUILD.md, "Voyager 3", 8): a group's stored corpus ranges in article order through one
+ * window in the Rust edge (`edge-bench --group`, the window carried from range to range) and the batch store into a
+ * scratch catalog, as the worker would store them: each range leased to a node named "replay", its batches stored
+ * at its resume point, the range finished when its last batch is in. Then a report: releases finished, complete,
+ * partial and provisional, and the catalog's bytes per header (the NVMe tables apart from those on CHUNK_TABLESPACE).
+ *
+ *   node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--from A] [--to B] [--limit N] [--batch-chunks 5]
+ *
+ * Refuses a database not named *_replay or *_scratch. The rules are the catalog's current rule set
+ * (rule_sets, the groups' priorities): copy `groups` and `rule_sets` from the catalog for a faithful replay.
+ */
+export default class ReplayCorpus extends BaseCommand {
+  static commandName = 'replay:corpus'
+  static description =
+    "Replays a group's corpus ranges through the Rust edge and the batch store into a scratch catalog"
+  static options: CommandOptions = { startApp: true }
+
+  @flags.string({ description: 'The group (its directory under the corpus)', required: true })
+  declare group: string
+
+  @flags.string({ description: 'The corpus directory (default: CORPUS_DIR)' })
+  declare dir?: string
+
+  @flags.number({ description: 'Replay this many ranges at most (default: all stored)' })
+  declare limit?: number
+
+  @flags.number({ description: 'Skip ranges ending before this article (continue a replay)' })
+  declare from?: number
+
+  @flags.number({
+    description: "Skip ranges starting after this article (one window of the group's history)",
+  })
+  declare to?: number
+
+  @flags.number({
+    description: 'Chunks of 100k articles per batch, as the worker posts them',
+    default: 5,
+  })
+  declare batchChunks: number
+
+  @flags.string({
+    description:
+      'The edge-bench binary (default: EDGE_BENCH, edge/target/release/edge-bench, /usr/local/bin/edge-bench)',
+  })
+  declare bench?: string
+
+  @flags.boolean({
+    description:
+      'Leave the window open at the end (the last ranges provisional), to continue later',
+  })
+  declare noFinish: boolean
+
+  async run() {
+    const database = db.connection().getReadClient().client.config.connection.database as string
+    const intoCatalog = process.env.REPLAY_INTO_CATALOG === '1' && !TEST_DB.test(database)
+    if (!SCRATCH_DB.test(database) && !intoCatalog) {
+      this.logger.error(
+        `The replay writes the catalog: it only runs against a database named *_replay or *_scratch (this is "${database}"; REPLAY_INTO_CATALOG=1 allows it)`
+      )
+      this.exitCode = 1
+      return
+    }
+    const dir = this.dir ?? coordinatorConfig.corpus.dir
+    if (!dir) {
+      this.logger.error('No corpus directory: --dir or CORPUS_DIR')
+      this.exitCode = 1
+      return
+    }
+    const bench = await this.benchBinary()
+    if (!bench) {
+      this.logger.error(
+        'No edge-bench binary found: build it (cargo build --release --manifest-path edge/Cargo.toml) or set EDGE_BENCH'
+      )
+      this.exitCode = 1
+      return
+    }
+    const groupDir = join(dir, this.group)
+    const stored = await this.storedRanges(groupDir)
+    if (stored.length === 0) {
+      this.logger.error(`${groupDir}: no complete range`)
+      this.exitCode = 1
+      return
+    }
+    const nodeId = await this.nodeId()
+    const groupId = await this.groupId()
+    const ranges = await this.leaseRanges(groupId, nodeId, stored)
+    const rules = await RuleSets.current()
+    const rulesPath = join(tmpdir(), `replay-rules-${process.pid}.json`)
+    await writeFile(rulesPath, JSON.stringify(rules))
+    this.logger.info(
+      `${this.group}: ${ranges.length} ranges (${ranges[0].start}–${ranges[ranges.length - 1].end}) through ${bench}, rules v${rules.version}, into ${database}.`
+    )
+
+    // edge-bench walks the group directory in article order itself; --from, --to and --limit cut its list as ours.
+    const args = [
+      '--group',
+      groupDir,
+      '--rules',
+      rulesPath,
+      '--batch-chunks',
+      String(this.batchChunks),
+      '--wire',
+      '-',
+    ]
+    if (this.from) args.push('--from', String(this.from))
+    if (this.to) args.push('--to', String(this.to))
+    if (this.limit) args.push('--limit', String(this.limit))
+    if (this.noFinish) args.push('--no-finish')
+    const started = Date.now()
+    const child = spawn(bench, args, { stdio: ['ignore', 'pipe', 'inherit'] })
+    const totals = {
+      batches: 0,
+      headers: 0,
+      releases: 0,
+      newReleases: 0,
+      finalized: 0,
+      candidates: 0,
+      storeMs: 0,
+    }
+    let current: StoredRange | null = null
+    const storeLine = async (line: string) => {
+      const batch = fromWire(JSON.parse(line) as WireBatch)
+      if (!current || batch.from < current.start || batch.from > current.end) {
+        current = ranges.find((r) => batch.from >= r.start && batch.from <= r.end) ?? null
+        if (!current) throw new Error(`Batch at ${batch.from} is in no leased range`)
+      }
+      const at = Date.now()
+      const result = await BatchStore.store(current.id, nodeId, batch)
+      totals.storeMs += Date.now() - at
+      totals.batches++
+      totals.headers += batch.headers
+      totals.releases += result.releases
+      totals.newReleases += result.newReleases
+      totals.finalized += result.finalized
+      totals.candidates += result.candidates
+      if (batch.next > current.end) {
+        await Leases.finish(current.id, nodeId)
+        const seconds = (Date.now() - started) / 1000
+        this.logger.info(
+          `  range ${current.start}–${current.end} done: ${totals.headers.toLocaleString('en')} headers so far in ${seconds.toFixed(0)} s ` +
+            `(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} releases stored (${totals.finalized} finished later), store ${(totals.storeMs / 1000).toFixed(0)} s`
+        )
+      }
+    }
+    // Lines pulled as the store keeps up: the child's stdout only moves when a chunk is read, so batches don't pile up.
+    let rest = ''
+    let failure: Error | null = null
+    try {
+      for await (const chunk of child.stdout as AsyncIterable<Buffer>) {
+        rest += chunk.toString('utf8')
+        let nl: number
+        while ((nl = rest.indexOf('\n')) >= 0) {
+          const line = rest.slice(0, nl)
+          rest = rest.slice(nl + 1)
+          if (line.length > 0) await storeLine(line)
+        }
+      }
+      if (rest.trim().length > 0) await storeLine(rest)
+    } catch (error: any) {
+      failure = error
+      child.kill('SIGTERM')
+    }
+    const code = await new Promise<number | null>((done) =>
+      child.exitCode !== null ? done(child.exitCode) : child.on('exit', done)
+    )
+    if (failure) {
+      this.logger.error(`Replay failed: ${failure.message}`)
+      this.exitCode = 1
+      return
+    }
+    if (code !== 0) {
+      this.logger.error(`edge-bench exited with ${code}`)
+      this.exitCode = 1
+      return
+    }
+    await this.report(groupId, ranges, Date.now() - started)
+  }
+
+  private async benchBinary(): Promise<string | null> {
+    const candidates = [
+      this.bench,
+      process.env.EDGE_BENCH,
+      resolve('edge/target/release/edge-bench'),
+      '/usr/local/bin/edge-bench',
+    ].filter((p): p is string => !!p)
+    for (const path of candidates) {
+      if (
+        await access(path).then(
+          () => true,
+          () => false
+        )
+      )
+        return path
+    }
+    return null
+  }
+
+  /** The group directory's complete ranges (a range.json), in article order, cut by --from, --to and --limit. */
+  private async storedRanges(groupDir: string): Promise<{ start: number; end: number }[]> {
+    const names = await readdir(groupDir).catch(() => [] as string[])
+    const ranges: { start: number; end: number }[] = []
+    for (const name of names) {
+      const m = /^(\d+)-(\d+)$/.exec(name)
+      if (!m) continue
+      const json = await readFile(join(groupDir, name, 'range.json'), 'utf8').catch(() => null)
+      if (!json) continue
+      ranges.push({ start: Number(m[1]), end: Number(m[2]) })
+    }
+    ranges.sort((a, b) => a.start - b.start)
+    let chosen = this.from ? ranges.filter((r) => r.end >= this.from!) : ranges
+    if (this.to) chosen = chosen.filter((r) => r.start <= this.to!)
+    if (this.limit) chosen = chosen.slice(0, this.limit)
+    return chosen
+  }
+
+  private async nodeId(): Promise<number> {
+    const row = await db.from('nodes').where('name', NODE_NAME).first()
+    if (row) return Number(row.id)
+    const [inserted] = await db.table('nodes').insert({ name: NODE_NAME }).returning('id')
+    return Number(inserted.id)
+  }
+
+  /** The group's row (inserted after the last priority when the scratch catalog doesn't have it). */
+  private async groupId(): Promise<number> {
+    const row = await db.from('groups').where('name', this.group).first()
+    if (row) return Number(row.id)
+    const { rows } = await db.rawQuery(`SELECT coalesce(max(priority), 0) + 1 AS next FROM groups`)
+    const [inserted] = await db
+      .table('groups')
+      .insert({ name: this.group, priority: Number(rows[0].next), active: true })
+      .returning('id')
+    this.logger.info(`Group ${this.group} added with priority ${rows[0].next}.`)
+    return Number(inserted.id)
+  }
+
+  /** The ranges as leased to the replay node, from their start (an earlier replay's rows are taken over). */
+  private async leaseRanges(
+    groupId: number,
+    nodeId: number,
+    stored: { start: number; end: number }[]
+  ): Promise<StoredRange[]> {
+    const out: StoredRange[] = []
+    const now = new Date()
+    for (const r of stored) {
+      const lease = {
+        status: 'leased',
+        node_id: nodeId,
+        next_id: r.start,
+        headers: 0,
+        posted_low: null,
+        posted_high: null,
+        leased_at: now,
+        heartbeat_at: now,
+        done_at: null,
+        error: null,
+        capture: true,
+      }
+      const existing = await db
+        .from('ranges')
+        .where('group_id', groupId)
+        .where('start_id', r.start)
+        .first()
+      if (existing) {
+        await db
+          .from('ranges')
+          .where('id', existing.id)
+          .update({ ...lease, end_id: r.end, attempts: Number(existing.attempts) + 1 })
+        out.push({ id: Number(existing.id), start: r.start, end: r.end })
+        continue
+      }
+      const [inserted] = await db
+        .table('ranges')
+        .insert({
+          ...lease,
+          group_id: groupId,
+          start_id: r.start,
+          end_id: r.end,
+          priority: 0,
+          attempts: 1,
+        })
+        .returning('id')
+      out.push({ id: Number(inserted.id), start: r.start, end: r.end })
+    }
+    return out
+  }
+
+  /** What the replay left in the catalog (this group's releases; the sizes are the whole catalog's), per article. */
+  private async report(groupId: number, ranges: StoredRange[], elapsedMs: number) {
+    const ids = ranges.map((r) => r.id)
+    const { rows: counts } = await db.rawQuery(
+      `SELECT count(*)::bigint AS releases,
+              count(settled_at)::bigint AS finished,
+              count(*) FILTER (WHERE settled_at IS NULL)::bigint AS provisional,
+              count(*) FILTER (WHERE edge)::bigint AS edges,
+              count(*) FILTER (WHERE complete)::bigint AS complete,
+              count(*) FILTER (WHERE complete = false)::bigint AS incomplete,
+              count(*) FILTER (WHERE complete IS NULL)::bigint AS undecided,
+              count(*) FILTER (WHERE readable)::bigint AS readable,
+              count(*) FILTER (WHERE has_par2)::bigint AS with_par2,
+              coalesce(sum(files), 0)::bigint AS files,
+              coalesce(sum(segments), 0)::bigint AS segments,
+              coalesce(sum(bytes), 0)::bigint AS bytes
+       FROM releases WHERE group_id = ?`,
+      [groupId]
+    )
+    const { rows: sources } = await db.rawQuery(
+      `SELECT source, count(*)::bigint AS n FROM releases WHERE group_id = ? GROUP BY source ORDER BY source`,
+      [groupId]
+    )
+    const { rows: pending } = await db.rawQuery(
+      `SELECT count(*)::bigint AS n, coalesce(sum(segments), 0)::bigint AS segments, count(*) FILTER (WHERE jsonb_array_length(candidates) = 0)::bigint AS unprobable
+       FROM pending_releases WHERE group_id = ?`,
+      [groupId]
+    )
+    const { rows: headerRows } = await db.rawQuery(
+      `SELECT coalesce(sum(headers), 0)::bigint AS headers FROM ranges WHERE id = ANY(?::bigint[])`,
+      [ids]
+    )
+    const headers = Number(headerRows[0].headers)
+    const { rows: sizes } = await db.rawQuery(
+      `SELECT CASE WHEN c.relname LIKE 'release_chunks%' THEN 'release_chunks' ELSE c.relname END AS name,
+              sum(pg_total_relation_size(c.oid))::bigint AS bytes
+       FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+       WHERE n.nspname = 'public' AND c.relkind = 'r'
+         AND (c.relname IN ('releases', 'release_texts', 'posters', 'group_sets', 'probe_candidates', 'rule_counts', 'rule_samples', 'blocks', 'pending_releases')
+              OR c.relname LIKE 'release_chunks%')
+       GROUP BY 1 ORDER BY 2 DESC`
+    )
+    // On CHUNK_TABLESPACE (the HDD) in the catalog: the chunks (migration 3200), inspections and candidates (6000).
+    const hdd = new Set([
+      'release_chunks',
+      'probe_candidates',
+      'release_inspections',
+      'pending_releases',
+    ])
+    let nvme = 0
+    let other = 0
+    const c = counts[0]
+    const fmt = (n: unknown) => Number(n).toLocaleString('en')
+    this.logger.info(
+      `Replay of ${ranges.length} ranges: ${fmt(headers)} headers in ${(elapsedMs / 1000).toFixed(0)} s (${fmt(Math.round(headers / Math.max(1, elapsedMs / 1000)))}/s).`
+    )
+    this.logger.info(
+      `Releases ${fmt(c.releases)}: finished ${fmt(c.finished)}, provisional ${fmt(c.provisional)}, edges ${fmt(c.edges)}; ` +
+        `complete ${fmt(c.complete)}, incomplete ${fmt(c.incomplete)}, undecided ${fmt(c.undecided)}; readable ${fmt(c.readable)}, with par2 ${fmt(c.with_par2)}; ` +
+        `${fmt(c.files)} files, ${fmt(c.segments)} segments (${((Number(c.segments) / Math.max(1, headers)) * 100).toFixed(1)}% of the headers), ${(Number(c.bytes) / 1e12).toFixed(2)} TB.`
+    )
+    this.logger.info(
+      `By source: ${sources.map((s: any) => `${s.source}: ${fmt(s.n)}`).join(', ')}.`
+    )
+    this.logger.info(
+      `Pending (unnamed, outside the catalog): ${fmt(pending[0].n)} releases, ${fmt(pending[0].segments)} segments, ${fmt(pending[0].unprobable)} without a probe target.`
+    )
+    for (const s of sizes) {
+      const bytes = Number(s.bytes)
+      if (hdd.has(s.name)) other += bytes
+      else nvme += bytes
+      this.logger.info(
+        `  ${s.name}: ${(bytes / 1e6).toFixed(1)} MB, ${(bytes / Math.max(1, headers)).toFixed(2)} B/header`
+      )
+    }
+    this.logger.info(
+      `NVMe tables: ${(nvme / 1e6).toFixed(1)} MB, ${(nvme / Math.max(1, headers)).toFixed(2)} B/header; HDD tables (CHUNK_TABLESPACE): ${(other / 1e6).toFixed(1)} MB, ${(other / Math.max(1, headers)).toFixed(2)} B/header (the whole catalog's sizes over this group's headers: only the first group's are its own).`
+    )
+  }
+}

commands/rules_classes.ts

@@ -0,0 +1,54 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import RuleSets from '#services/rule_sets'
+import { classKey, describeClass } from '#lib/release/index'
+
+/**
+ * Publishes a rule set whose dropped classes are the pending ladder's (pending_classes.dropped; LOG.md (130)): the
+ * edge keeps unnamed releases of those classes out of the pending store from then on, their explored sample
+ * excepted. Run again as the record grows; unchanged classes publish nothing.
+ */
+export default class RulesClasses extends BaseCommand {
+  static commandName = 'rules:classes'
+  static description = "Publishes a rule set with the pending ladder's dropped classes"
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'Show what would be published, write nothing' })
+  declare dryRun: boolean
+
+  async run() {
+    const { rows } = await db.rawQuery(
+      `SELECT c.tool, g.name AS grp, c.klass, c.probed, c.named FROM pending_classes c JOIN groups g ON g.id = c.group_id WHERE c.dropped ORDER BY 1, 2, 3`
+    )
+    const next = new Set<string>(rows.map((r: any) => classKey(r.tool, r.grp, Number(r.klass))))
+    const { version, priorities: _p, ...current } = await RuleSets.current()
+    const before = new Set(current.droppedClasses ?? [])
+    const added = [...next].filter((c) => !before.has(c))
+    const removed = [...before].filter((c) => !next.has(c))
+    if (added.length === 0 && removed.length === 0) {
+      this.logger.info(
+        `Unchanged: rule set ${version} has the ${before.size} dropped classes already.`
+      )
+      return
+    }
+    for (const r of rows.slice(0, 40)) {
+      this.logger.info(
+        `  ${r.tool.padEnd(10)} ${r.grp.padEnd(32)} ${describeClass(Number(r.klass)).padEnd(26)} ${r.named} of ${r.probed} named`
+      )
+    }
+    if (this.dryRun) {
+      this.logger.info(
+        `Would publish ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
+      )
+      return
+    }
+    const published = await RuleSets.publish(
+      { ...current, droppedClasses: [...next].sort() },
+      `${next.size} dropped classes from the pending ladder`
+    )
+    this.logger.info(
+      `Rule set ${published}: ${next.size} dropped classes (${added.length} added, ${removed.length} removed).`
+    )
+  }
+}

commands/rules_signatures.ts

@@ -0,0 +1,52 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { readFile } from 'node:fs/promises'
+import RuleSets from '#services/rule_sets'
+import { signatureKey } from '#lib/rules/index'
+
+/**
+ * Sets the dropped upload signatures (REBUILD.md, rules and recovery) from a
+ * file of "tool<TAB>group" lines, e.g. voyager 1's learned ones:
+ *
+ *   psql … -At -F $'\t' -c "SELECT tool, group_name FROM upload_signatures WHERE dropped"
+ *
+ * Publishes a new rule set version (ranges record the version they ran with).
+ */
+export default class RulesSignatures extends BaseCommand {
+  static commandName = 'rules:signatures'
+  static description = 'Publishes a rule set with the dropped upload signatures from a file'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'File of "tool<TAB>group" lines (more columns and # lines are ignored), e.g. database/data/voyager1_signatures.tsv' })
+  declare file: string
+
+  @flags.string({ description: 'Note stored with the rule set version' })
+  declare note?: string
+
+  @flags.boolean({ description: 'Add to the current signatures instead of replacing them' })
+  declare add: boolean
+
+  async run() {
+    const lines = (await readFile(this.file, 'utf8')).split('\n').map((l) => l.trim()).filter((l) => l && !l.startsWith('#'))
+    const signatures = new Set<string>()
+    for (const line of lines) {
+      const [tool, group] = line.split('\t')
+      if (!tool || !group) throw new Error(`Not "tool<TAB>group": ${line}`)
+      signatures.add(signatureKey(tool, group))
+    }
+    const { version: _v, priorities: _p, ...current } = await RuleSets.current()
+    const before = new Set(current.droppedSignatures)
+    const next = this.add ? new Set([...before, ...signatures]) : signatures
+    if (next.size === before.size && [...next].every((s) => before.has(s))) {
+      this.logger.info(`Unchanged: ${before.size} dropped signatures already.`)
+      return
+    }
+    const version = await RuleSets.publish(
+      { ...current, droppedSignatures: [...next].sort() },
+      this.note ?? `${next.size} dropped signatures from ${this.file}`
+    )
+    const added = [...next].filter((s) => !before.has(s)).length
+    const removed = [...before].filter((s) => !next.has(s)).length
+    this.logger.info(`Rule set ${version}: ${next.size} dropped signatures (${added} added, ${removed} removed).`)
+  }
+}

commands/spots_inspect.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { SpotInspector } from '#services/spot_inspector'
+
+/** The spot inspector (REBUILD.md, spots): each spot's NZB read, its files listed and matched to releases. */
+export default class SpotsInspect extends BaseCommand {
+  static commandName = 'spots:inspect'
+  static description = "Reads each listed spot's NZB, lists its files and links them to the releases they match"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  async run() {
+    const inspector = new SpotInspector()
+    if (this.once) {
+      const round = await inspector.round()
+      this.logger.info(`${round.inspected} spots inspected, ${round.matched} matched releases, ${round.standalone} of their own.`)
+      inspector.stop()
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      inspector.stop()
+      await finished
+    })
+    await inspector.run()
+    this.logger.info('Spot inspector stopped.')
+    done!()
+  }
+}

commands/spots_read.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Spotter } from '#services/spotter'
+
+/** The spotter (REBUILD.md, spots): Spotnet's spots from free.pt's headers into `spots`, on one connection. */
+export default class SpotsRead extends BaseCommand {
+  static commandName = 'spots:read'
+  static description = "Reads Spotnet's spots (free.pt) into the spots table, verified, every few minutes"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  async run() {
+    const spotter = new Spotter()
+    if (this.once) {
+      const round = await spotter.round()
+      this.logger.info(`${round.read} articles read: ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
+      spotter.stop()
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      spotter.stop()
+      await finished
+    })
+    await spotter.run()
+    this.logger.info('Spotter stopped.')
+    done!()
+  }
+}

config/coordinator.ts

@@ -12,18 +12,178 @@ export default {
    * 2026-10-09).
    */
   chunkArticles: env.get('CHUNK_ARTICLES', 100_000),
+  /**
+   * The live tail (REBUILD.md, coordinator): new articles become ranges
+   * every `everySeconds` (one LIST ACTIVE per name prefix), leased and
+   * fetched before any history. A group gets a tail range once it has
+   * `minArticles` new (a chunk) or its oldest unplanned article waited
+   * `maxDelayMinutes`, so a quiet group doesn't make a range every round.
+   */
+  tail: {
+    everySeconds: env.get('TAIL_EVERY_SECONDS', 120),
+    minArticles: env.get('TAIL_MIN_ARTICLES', 100_000),
+    maxDelayMinutes: env.get('TAIL_MAX_DELAY_MINUTES', 15),
+    /**
+     * Before everything: the history's 0 (oldest) … 1000 (newest), the stored ranges' -2 and ranges raised to -1 (the
+     * short gaps between done blocks went before the tail at -1 and it fell ~4 h behind; LOG.md 2026-10-16 (70)).
+     */
+    priority: -3,
+  },
   /** A leased range without a heartbeat for this long goes back to the queue. */
   leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
-  /** NNTP connections the coordinator fetches headers with. */
+  /**
+   * The ranges group leases hand out (REBUILD.md design 1), by priority: the tail is negative, the capture windows
+   * 0–2 (2016, 2021, 2025), the history by its position in the group (0 oldest … 1000 newest: its first buckets
+   * share 0–2 with the windows) or 4000+. Bounds so a deployment can run one window first, with
+   * LEASE_CAPTURE_ONLY so the history's 0 isn't taken along (homelab 2026-10-26: 0..0 and capture only, the 2016
+   * window; LOG.md (118)); unbounded and any range by default.
+   */
+  leasePriorityMin: env.get('LEASE_MIN_PRIORITY', -1_000_000),
+  leasePriorityMax: env.get('LEASE_MAX_PRIORITY', 1_000_000),
+  leaseCaptureOnly: env.get('LEASE_CAPTURE_ONLY', false),
+  /**
+   * How long a range's stream waits for a chunk the prefetcher hasn't spooled yet (app/controllers/node_controller.ts).
+   * Long: a chunk that keeps coming in isn't given up (only a stalled one is), and some groups' chunks take 20+
+   * minutes (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
+   */
+  streamChunkWaitMs: env.get('STREAM_CHUNK_WAIT_MINUTES', 60) * 60 * 1000,
+  /** NNTP connections the coordinator fetches headers with: at most (its share, nntpShares), and at least while sharing. */
   nntpConnections: env.get('NNTP_CONNECTIONS', 4),
-  /** A reply with no data for this long fails (old ranges answer slowly). */
-  nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 10 * 60 * 1000),
+  nntpMinConnections: env.get('NNTP_MIN_CONNECTIONS', 8),
+  /**
+   * The account's connections, shared by the coordinator and the prober by priority (app/services/nntp_shares.ts):
+   * all of them less the reserved ones (the spotter's and the spot inspector's 1 each, the coordinator's plain one,
+   * and 2 for a hedged request's or a reconnect's new connection while the old one closes).
+   */
+  nntpShares: {
+    total: env.get('NNTP_ACCOUNT_CONNECTIONS', 50) - env.get('NNTP_RESERVED_CONNECTIONS', 5),
+    everyMs: 15_000,
+    /** Headers before probes: history takes what it can use, the prober the rest. */
+    coordinatorPriority: 2,
+    proberPriority: 1,
+  },
+  /**
+   * A reply with no data for this long fails, and its connection is replaced.
+   * Eweka sometimes stops answering on long-lived connections (all 40 quiet,
+   * a fresh one answers in seconds; LOG.md 2026-10-09 (14)): at 10 minutes
+   * the prefetcher nearly stopped. Data arrives within seconds otherwise.
+   */
+  nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 2 * 60 * 1000),
+  /**
+   * A request that got less than this in the last minute (kB) is given up,
+   * its connection closed, and the chunk fetched again: a reply that trickles
+   * (6,204 → 6,229 kB in a minute) never goes quiet long enough for
+   * nntpTimeoutMs, and kept a worker waiting on one chunk for an hour. Not a
+   * deadline: some groups send steadily but slowly (a.b.0day.stuffz ~400 kB a
+   * minute, 5–7 minutes a chunk), and a fixed 90–360 s gave those up again
+   * and again (LOG.md 2026-10-10 (3)).
+   */
+  nntpStallBytesPerMinute: env.get('NNTP_STALL_KB_PER_MINUTE', 100) * 1000,
+  /**
+   * A chunk not fetched after this long is asked again on another
+   * connection; the first answer wins and the other connection is closed.
+   * Eweka stalls single requests: the same chunk took 27–52 s on one
+   * connection and 1–9 s on another at the same moment; median ~3 s
+   * (LOG.md 2026-10-09 (15)). Measured in production (40 + 6 connections vs
+   * 40, 15 min each): 280k vs 273k articles/s, within the noise; the second
+   * request won half the time. Off by default.
+   */
+  nntpHedgeAfterMs: env.get('NNTP_HEDGE_AFTER_SECONDS', 10) * 1000,
+  /** Of nntpConnections, kept for those second requests (the prefetcher's loops are the rest). 0: no hedging. */
+  nntpHedgeConnections: env.get('NNTP_HEDGE_CONNECTIONS', 0),
+  /**
+   * A connection is closed after this long (when it's free), and a fresh one
+   * opened. 0: never. Requests were as often slow (>10 s) on connections
+   * under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)).
+   */
+  nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000,
+  /**
+   * A connection with nothing to do for this long is closed, and opened again
+   * when there's work. 0: kept. Eweka drops connections that stay quiet, and
+   * the next command on one failed ("Spot inspection failed: NNTP connection
+   * closed"); idle ones also take the account's connections from downloads.
+   */
+  nntpIdleCloseMs: env.get('NNTP_IDLE_CLOSE_SECONDS', 60) * 1000,
+  probe: {
+    /**
+     * At most (its share, nntpShares): NNTP connections the prober reads articles with, besides the
+     * coordinator's: the account has 50, the coordinator 40 + 1 plain, and a
+     * couple stay free for downloads. ~7 MB/s each (~10 articles/s).
+     */
+    connections: env.get('PROBE_CONNECTIONS', 6),
+    /** Of them, always (its share's minimum; nntpShares). */
+    minConnections: env.get('PROBE_MIN_CONNECTIONS', 2),
+    /** No claims while the filesystem under `/` (the NVMe, in the container) has less free (app/services/disk_space.ts). 0: off. */
+    minFreeGb: env.get('PROBE_MIN_FREE_GB', 0),
+    /** Candidates claimed at a time. */
+    batch: 400,
+    /** BODY commands in flight per connection (pipelined; XoverClient.bodies). */
+    pipeline: env.get('PROBE_PIPELINE', 8),
+    /**
+     * The pending ladder (REBUILD.md decision f; app/services/pending_ladder.ts): the share of each claim taken
+     * from pending_releases at least (the catalog's own candidates come first, the rest of the claim is theirs).
+     */
+    pendingShare: env.get('PENDING_SHARE', 0.25),
+    /** A signature (tool, group, poster) with this many climbs and at most pendingMaxNamed of them named is dropped. */
+    pendingMinProbes: env.get('PENDING_MIN_PROBES', 100),
+    pendingMaxNamed: env.get('PENDING_MAX_NAMED', 0.005),
+  },
+  /**
+   * The catalog's tablespace may grow to this many GB (app/services/catalog_space.ts); above it only the tail and
+   * the stored ranges are leased and prefetched. 0: no cap.
+   */
+  catalogMaxNvmeGb: env.get('CATALOG_MAX_NVME_GB', 0),
+  /**
+   * Spotnet spots (app/services/spotter.ts, spot_inspector.ts): the group the spots are posted in, how many of
+   * its last articles the spotter reads the first time (~215 spots a day: ~3 months), how often it looks again,
+   * the most articles an NZB may take, and how long an inspected spot whose files match no release waits for
+   * the header pass before it becomes a release of its own.
+   */
+  spots: {
+    group: env.get('SPOT_GROUP', 'free.pt'),
+    backfill: env.get('SPOT_BACKFILL', 20_000),
+    everySeconds: env.get('SPOT_EVERY_SECONDS', 120),
+    maxNzbSegments: env.get('SPOT_MAX_NZB_SEGMENTS', 64),
+    standaloneAfterHours: env.get('SPOT_STANDALONE_AFTER_HOURS', 24),
+  },
+  /** Stored XOVER chunks of ranges marked `capture`, for the grouping sweeps (app/services/corpus.ts). Off when unset. */
+  corpus: {
+    dir: env.get('CORPUS_DIR'),
+    /**
+     * Capture ranges fetched into the corpus by connections the spool leaves idle, this many at a time (8, 2 connections each, left 20 of 40 idle): the
+     * spool only fetches what workers lease soon, and the tail comes first, so after a reset the capture
+     * ranges waited behind it for hours (1,400 marked, none stored, 34 of 40 connections idle; LOG.md 2026-10-12 (27)).
+     */
+    ahead: env.get('CORPUS_AHEAD', 16),
+    /**
+     * Stored without the lines the edge worker drops on their own (crossposts another indexed group owns, obfuscated
+     * subjects; ~48% of the indexed ranges' headers), what was left out in each chunk's `.drops`: twice the history
+     * in the same space (LOG.md 2026-10-23 (101)). Off: as the news server sent them.
+     */
+    filter: env.get('CORPUS_FILTER', true),
+  },
+  /**
+   * The group windows' snapshots (REBUILD.md, "Voyager 3: window indexing", 5; app/services/group_leases.ts): a
+   * file a group on bulk, next to the spool by default.
+   */
+  snapshotDir: env.get('SNAPSHOT_DIR', `${env.get('SPOOL_DIR', app.tmpPath('spool'))}/snapshots`),
   spool: {
     dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
     /** Ranges kept fetched ahead, per active lease. */
-    aheadPerLease: env.get('SPOOL_AHEAD_PER_LEASE', 2),
+    aheadPerLease: env.get('SPOOL_AHEAD_PER_LEASE', 4),
     /** At least this many ranges ahead, so the first leases start at once. */
     minAhead: env.get('SPOOL_MIN_AHEAD', 2),
     maxBytes: env.get('SPOOL_MAX_GB', 10) * 1e9,
+    /**
+     * At most this many connections on one range, and on one group, at a
+     * time (0: no limit): spreads the requests over ranges and groups instead
+     * of all on the next chunks of the first ranges. Homelab, 40 connections,
+     * 12 min each, alternated: unlimited (on ~10 ranges) 281k and 167k
+     * articles/s, 2 per range and 4 per group (on ~22) 319k and 296k, with
+     * shorter chunk times (11.5 vs 13–24 s; LOG.md 2026-10-09 (18)). Needs
+     * targets for it: aheadPerLease 4.
+     */
+    perRange: env.get('PREFETCH_PER_RANGE', 2),
+    perGroup: env.get('PREFETCH_PER_GROUP', 4),
   },
 }

config/static.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from '@adonisjs/static'
+
+/**
+ * Configuration options to tweak the static files middleware.
+ * The complete set of options are documented on the
+ * official documentation website.
+ *
+ * https://docs.adonisjs.com/guides/static-assets
+ */
+const staticServerConfig = defineConfig({
+  enabled: true,
+  etag: true,
+  lastModified: true,
+  dotFiles: 'ignore',
+})
+
+export default staticServerConfig

database/data/voyager1_signatures.tsv

@@ -0,0 +1,30 @@
+# Voyager 1 upload_signatures WHERE dropped, 2026-10-09: tool, group, probed, named
+other	alt.binaries.ath	10988	24
+ngPost	alt.binaries.ath	3843	0
+hex32	alt.binaries.test	1890	2
+ngPost	alt.binaries.a51	1516	0
+nyuu	alt.binaries.a51	700	0
+nyuu	alt.binaries.mom	656	0
+hex32	alt.binaries.boneless	453	1
+nyuu	alt.binaries.misc	356	0
+nyuu	alt.binaries.games.encrypted	353	0
+other	alt.binaries.etc	317	1
+other	alt.binaries.misc	310	0
+nyuu	alt.binaries.warez	292	0
+other	alt.binaries.usenet2day	255	0
+other	alt.binaries.ijsklontje	238	0
+ngPost	alt.binaries.bloaf	237	0
+nyuu	alt.binaries.nl	196	0
+other	alt.binaries.archive.encrypted	189	0
+nyuu	alt.binaries.frogs	172	0
+other	alt.binaries.encrypted	160	0
+yuun	alt.binaries.nl	152	0
+ngPost	alt.binaries.nl	149	0
+other	alt.binaries.bloaf	134	0
+nyuu	alt.binaries.erotica	119	0
+other	alt.binaries.warez	119	0
+other	alt.binaries.ftn	107	0
+nyuu	alt.binaries.ijsklontje	106	0
+random	alt.binaries.newznzb.delta	103	0
+ngPost	alt.binaries.mom	102	0
+random	alt.binaries.newznzb.charlie	101	0

database/migrations/1770000001200_release_category.ts

@@ -0,0 +1,39 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A release's Newznab category from its name (REBUILD.md, part 9): voyager
+ * 1's release_category() without the archive contents (phase 2 has them).
+ * TV (5030/5040/5045), movies (2030/2040/2045), audio (3010/3040),
+ * books (7020), otherwise 8010.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(String.raw`
+      CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
+        SELECT CASE
+          -- TV: S01E02, 1x02, or a season pack (S01).
+          WHEN name ~* '(^|[._ -])(S\d{1,2}E\d{1,3}|\d{1,2}x\d{2})([._ -]|$)' OR name ~* '[._ -]S\d{1,2}([._ -]|$)' THEN
+            CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
+                 WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
+          -- Movies: a year plus a source or quality marker.
+          WHEN name ~* '(^|[._ (-])(19|20)\d{2}([._ )-]|$)'
+               AND name ~* '(480p|576p|720p|1080p|2160p|blu-?ray|web-?dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.?26[45]|hevc|remux)' THEN
+            CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
+                 WHEN name ~* '720p|1080p|blu-?ray' THEN 2040 ELSE 2030 END
+          -- Audio: lossless, mp3, or scene music names (-WEB-2026-GROUP).
+          WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
+          WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)\d{2}-[A-Za-z0-9]+$' THEN 3010
+          WHEN name ~* '(^|[._ -])(epub|ebook|mobi)([._ -]|$)' THEN 7020
+          ELSE 8010
+        END
+      $$`)
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS category integer GENERATED ALWAYS AS (release_category(name)) STORED`)
+    // Newznab and the web interface list a category newest first.
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS releases_category_posted_idx ON releases (category, posted_at DESC)`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS category`)
+    this.schema.raw(`DROP FUNCTION IF EXISTS release_category(text)`)
+  }
+}

database/migrations/1770000001300_groups_tail.ts

@@ -0,0 +1,15 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A group's live tail can be paused (its history and its place in the owner
+ * rule stay): e.g. to let other groups' history go first.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE groups ADD COLUMN IF NOT EXISTS tail boolean NOT NULL DEFAULT true`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE groups DROP COLUMN IF EXISTS tail`)
+  }
+}

database/migrations/1770000001400_release_uploads.ts

@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Re-uploads as releases of their own (REBUILD.md, decision d): releases
+ * keep the identity key their files gave (identity_key; release_key is that
+ * for the first upload, then key:2, key:3, …), the group that owns their
+ * articles, and their posting window (posted_at … last_posted_at). A batch's
+ * chunk joins a release with its key and group whose window is within
+ * UPLOAD_GAP_MS, else starts one.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS identity_key varchar(40)`)
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS group_id integer`)
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS last_posted_at timestamptz`)
+    this.schema.raw(`UPDATE releases SET identity_key = release_key, last_posted_at = posted_at WHERE identity_key IS NULL`)
+    this.schema.raw(`ALTER TABLE releases ALTER COLUMN identity_key SET NOT NULL`)
+    this.schema.raw(`ALTER TABLE releases ALTER COLUMN release_key TYPE varchar(48)`)
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS releases_identity_idx ON releases (identity_key, group_id)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX IF EXISTS releases_identity_idx`)
+    this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS identity_key, DROP COLUMN IF EXISTS group_id, DROP COLUMN IF EXISTS last_posted_at`)
+  }
+}

database/migrations/1770000001500_ranges_plain_chunks.ts

@@ -0,0 +1,18 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Chunks of a range to fetch without compression: Eweka sends some ranges'
+ * compressed XOVER truncated, the same bytes every time, while the plain
+ * reply is whole (LOG.md 2026-10-09 (11)). A worker reports such a chunk;
+ * its start goes here and the prefetcher fetches it plain.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS plain_chunks bigint[] NOT NULL DEFAULT '{}'`)
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS error text`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS plain_chunks, DROP COLUMN IF EXISTS error`)
+  }
+}

database/migrations/1770000001600_tsm_system_rows.ts

@@ -0,0 +1,12 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** TABLESAMPLE SYSTEM_ROWS: the stats' sample of the catalog at a fixed size (a percentage read 30 s of random pages on the HDD at 7M releases). */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`CREATE EXTENSION IF NOT EXISTS tsm_system_rows`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP EXTENSION IF EXISTS tsm_system_rows`)
+  }
+}

database/migrations/1770000001700_release_inspections.ts

@@ -0,0 +1,38 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * Phase 2's probe results (REBUILD.md, part 6): per release and probe kind
+ * what its article showed (yEnc name, content type, a few names inside,
+ * encryption), in CHUNK_TABLESPACE (the HDD) when set: they're read per
+ * release, not searched. On the catalog only what search needs: whether the
+ * release is passworded and its best inner name.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    const where = tablespace ? ` TABLESPACE ${tablespace}` : ''
+    this.schema.raw(`
+      CREATE TABLE release_inspections (
+        release_id bigint NOT NULL,
+        kind varchar(8) NOT NULL,
+        result jsonb NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (release_id, kind)${tablespace ? ` USING INDEX TABLESPACE ${tablespace}` : ''}
+      )${where}`)
+    this.schema.alterTable('releases', (table) => {
+      table.boolean('passworded').nullable()
+      table.string('inner_name', 255).nullable()
+      table.timestamp('inspected_at', { useTz: true }).nullable()
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => {
+      table.dropColumn('passworded')
+      table.dropColumn('inner_name')
+      table.dropColumn('inspected_at')
+    })
+    this.schema.dropTable('release_inspections')
+  }
+}

database/migrations/1770000001800_release_inspections_per_candidate.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** One inspection per probe candidate: a release can have several of a kind (two RAR sets). */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`DELETE FROM release_inspections`)
+    this.schema.raw(`ALTER TABLE release_inspections DROP CONSTRAINT release_inspections_pkey`)
+    this.schema.raw(`ALTER TABLE release_inspections ADD COLUMN candidate_id bigint NOT NULL`)
+    this.schema.raw(`ALTER TABLE release_inspections ADD PRIMARY KEY (release_id, candidate_id)`)
+    // Their results went with the deleted rows: probed again.
+    this.schema.raw(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'done'`)
+    this.schema.raw(`UPDATE releases SET passworded = NULL, inner_name = NULL, inspected_at = NULL WHERE inspected_at IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE release_inspections DROP CONSTRAINT release_inspections_pkey`)
+    this.schema.raw(`ALTER TABLE release_inspections DROP COLUMN candidate_id`)
+    this.schema.raw(`ALTER TABLE release_inspections ADD PRIMARY KEY (release_id, kind)`)
+  }
+}

database/migrations/1770000001900_releases_inner_name_trgm.ts

@@ -0,0 +1,18 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Search matches the prober's inner names too (SearchService): a trigram
+ * index on them, only where there is one. Built concurrently: the catalog
+ * keeps taking batches meanwhile.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_inner_name_trgm ON releases USING gin (inner_name gin_trgm_ops) WHERE inner_name IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_inner_name_trgm`)
+  }
+}

database/migrations/1770000002000_releases_sort_indexes.ts

@@ -0,0 +1,25 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Indexes that match the newest-first sort (SearchService: posted_at DESC
+ * NULLS LAST, id DESC). The old ones (posted_at DESC: nulls first) didn't,
+ * so the default page sorted all 68M releases: 23 s. Built concurrently, the
+ * old ones dropped after.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`)
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_category_newest_idx ON releases (category, posted_at DESC NULLS LAST, id DESC)`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_posted_at_idx`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_category_posted_idx`)
+  }
+
+  async down() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_posted_at_idx ON releases (posted_at DESC)`)
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_category_posted_idx ON releases (category, posted_at DESC)`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_newest_idx`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_category_newest_idx`)
+  }
+}

database/migrations/1770000002100_ranges_capture.ts

@@ -0,0 +1,12 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** Ranges whose XOVER chunks are kept in the corpus (CORPUS_DIR) for the grouping sweeps. */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS capture boolean NOT NULL DEFAULT false`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS capture`)
+  }
+}

database/migrations/1770000002200_par2_merge.ts

@@ -0,0 +1,60 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The par2 merge (REBUILD.md, phase 2; app/services/par2_merger.ts): what the
+ * prober read that ties files to a par2 set, and the merges made from it.
+ * - file_hashes: per probed article that starts a file, the MD5 of its first 16 KiB and the file's size.
+ * - par2_files: per par2 recovery set, every file it describes (FileDesc: the same 16 KiB MD5, length, name).
+ * - par2_sets: which releases hold a par2 file of a set.
+ * - release_merges: releases merged into another, by set.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE file_hashes (
+        candidate_id bigint PRIMARY KEY,
+        release_id bigint NOT NULL,
+        hash16k char(32) NOT NULL,
+        size bigint NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+    this.schema.raw(`CREATE INDEX file_hashes_hash_idx ON file_hashes (hash16k)`)
+    this.schema.raw(`CREATE INDEX file_hashes_release_idx ON file_hashes (release_id)`)
+    this.schema.raw(`CREATE INDEX file_hashes_created_idx ON file_hashes (created_at)`)
+    this.schema.raw(`
+      CREATE TABLE par2_files (
+        set_id char(32) NOT NULL,
+        file_id char(32) NOT NULL,
+        hash16k char(32) NOT NULL,
+        length bigint NOT NULL,
+        name text NOT NULL,
+        PRIMARY KEY (set_id, file_id)
+      )`)
+    this.schema.raw(`CREATE INDEX par2_files_hash_idx ON par2_files (hash16k)`)
+    this.schema.raw(`
+      CREATE TABLE par2_sets (
+        set_id char(32) NOT NULL,
+        release_id bigint NOT NULL,
+        files integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (set_id, release_id)
+      )`)
+    this.schema.raw(`CREATE INDEX par2_sets_release_idx ON par2_sets (release_id)`)
+    this.schema.raw(`CREATE INDEX par2_sets_created_idx ON par2_sets (created_at)`)
+    this.schema.raw(`
+      CREATE TABLE release_merges (
+        id bigserial PRIMARY KEY,
+        from_id bigint NOT NULL,
+        from_key varchar(48) NOT NULL,
+        into_id bigint NOT NULL,
+        set_id char(32) NOT NULL,
+        files integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+    this.schema.raw(`CREATE INDEX release_merges_into_idx ON release_merges (into_id)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS file_hashes, par2_files, par2_sets, release_merges`)
+  }
+}

database/migrations/1770000002300_probe_cache.ts

@@ -0,0 +1,25 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * What a probed article showed, by message-id (app/services/prober.ts): an
+ * article's inspection doesn't depend on the grouping rules, so a catalog
+ * reset (which empties the probe queue) needn't read it from the news server
+ * again. `version` is PROBE_VERSION when it was inspected: a change to the
+ * inspection makes older rows stale. Not truncated by a reset.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE probe_cache (
+        message_id text PRIMARY KEY,
+        status varchar(8) NOT NULL,
+        result jsonb,
+        version integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS probe_cache`)
+  }
+}

database/migrations/1770000002400_probe_candidates_kind_index.ts

@@ -0,0 +1,19 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The prober claims waiting candidates by kind from random points of the
+ * queue (Prober.claim): by the id index alone a slice filtered ~700k rows for
+ * the rarer kinds (0.37 s a slice, 20 slices and up to 5 tiers a claim, the
+ * lanes waiting meanwhile; LOG.md 2026-10-12 (17)). Built concurrently.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_kind_waiting_idx ON probe_candidates (kind, id) WHERE status = 'waiting'`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_kind_waiting_idx`)
+  }
+}

database/migrations/1770000002500_probe_candidates_ready.ts

@@ -0,0 +1,30 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Candidates the prober may claim (Prober.claim): `ready` once their release
+ * settled (the settler sets it), `unreadable` when that release had no
+ * readable name then. The claims joined releases for both and started at
+ * random points of the queue; in its newer half nothing had settled, so a
+ * slice scanned to the end (0.3 s a slice, 20 slices, 5 tiers: the prober
+ * stood still for minutes per claim; LOG.md 2026-10-12 (20)). With an index
+ * over the claimable rows only.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`ALTER TABLE probe_candidates ADD COLUMN IF NOT EXISTS ready boolean NOT NULL DEFAULT false`)
+    this.schema.raw(`ALTER TABLE probe_candidates ADD COLUMN IF NOT EXISTS unreadable boolean NOT NULL DEFAULT false`)
+    this.schema.raw(
+      `UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
+       WHERE r.id = pc.release_id AND r.settled_at IS NOT NULL AND NOT pc.ready`
+    )
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_ready_idx ON probe_candidates (kind, unreadable, id) WHERE status = 'waiting' AND ready`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_kind_waiting_idx`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_ready_idx`)
+    this.schema.raw(`ALTER TABLE probe_candidates DROP COLUMN IF EXISTS ready, DROP COLUMN IF EXISTS unreadable`)
+  }
+}

database/migrations/1770000002600_inner_stem_merge.ts

@@ -0,0 +1,30 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The merge by inner name (app/services/par2_merger.ts, InnerStemMerger):
+ * obfuscated uploads post each file under a subject name of its own, but the
+ * yEnc name inside its first article shares a random stem with the rest of
+ * the upload ("[1/7] - "471044d8…"" holds "bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001";
+ * hep, hephaestus, gyges: 31,000 one-file releases; LOG.md 2026-10-12 (32)).
+ * - An index over a lone inspection's stem: the yEnc name up to its first dot, when 16–64 letters and digits.
+ * - release_merges: which kind of merge (par2 or inner), and room for a stem in set_id.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS release_inspections_inner_stem_idx ON release_inspections
+       ((substring(result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.'))) WHERE kind = 'lone'`
+    )
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS release_inspections_created_idx ON release_inspections (created_at)`)
+    this.schema.raw(`ALTER TABLE release_merges ALTER COLUMN set_id TYPE varchar(64)`)
+    this.schema.raw(`ALTER TABLE release_merges ADD COLUMN IF NOT EXISTS via varchar(8) NOT NULL DEFAULT 'par2'`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS release_inspections_inner_stem_idx`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS release_inspections_created_idx`)
+    this.schema.raw(`ALTER TABLE release_merges DROP COLUMN IF EXISTS via`)
+  }
+}

database/migrations/1770000002700_release_inspections_catalog_tablespace.ts

@@ -0,0 +1,27 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * release_inspections in the catalog's tablespace (default_tablespace: nvme
+ * on homelab), not CHUNK_TABLESPACE: the prober writes it every flush and
+ * the merger looks stems up in it (release_inspections_inner_stem_idx), so it
+ * isn't only read per release any more (migration 1700). Its flushes took
+ * 35–83 s a minute with the table on the HDD (LOG.md 2026-10-13 (43)).
+ * Homelab was moved by hand first; here a no-op when it's there already, or
+ * without a default tablespace (tests).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const { rows } = await db.rawQuery(
+        `SELECT current_setting('default_tablespace') AS target,
+                (SELECT t.spcname FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.relname = 'release_inspections') AS current`
+      )
+      const { target, current } = rows[0]
+      if (!target || current === target) return
+      await db.rawQuery(`ALTER TABLE release_inspections SET TABLESPACE ${target}`)
+      await db.rawQuery(`ALTER INDEX release_inspections_pkey SET TABLESPACE ${target}`)
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000002800_spots.ts

@@ -0,0 +1,96 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Spotnet spots (REBUILD.md, spots; after voyager 1's D29), in two processes:
+ * - the spotter (`spots:read`) reads free.pt's headers, verifies each spot's
+ *   signature from its full header (X-XML) and keeps it in `spots`, with a
+ *   password its title or description names;
+ * - the spot inspector (`spots:inspect`) reads each spot's compressed NZB
+ *   (alt.binaries.ftd), keeps its files in `spot_files`, and matches them to
+ *   the releases the header pass built (the same release identity), linking
+ *   them in `release_spots`. A spot whose files match no release becomes a
+ *   release of its own (source `spot`) a day after its inspection (until
+ *   the header pass builds its releases after all: migration 2900).
+ * The NZB itself is kept (gzip) for such a release's downloads.
+ * Passwords are kept apart from the catalog in `release_passwords`, one row
+ * per release and password, with where it came from (the NZB's meta, the
+ * spot's title or its text); the catalog only says `passworded`.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE spots (
+        message_id text PRIMARY KEY,
+        article_number bigint,
+        title text NOT NULL,
+        tag text,
+        poster text NOT NULL,
+        category integer,
+        subcats jsonb,
+        size bigint,
+        posted_at timestamptz,
+        key_id integer,
+        spotter_id text,
+        description text,
+        website text,
+        nzb_segments text[],
+        -- The NZB as read (gzip): a release the spot makes of its own has no chunks to build one from.
+        nzb bytea,
+        password text,
+        password_source varchar(16),
+        -- listed: verified, waiting for the inspector; inspected, no-nzb, dummy, nzb-gone, nzb-unreadable: after it;
+        -- moderation: a delete/dispose message; removed: disposed of by a moderator or its spotter.
+        status varchar(16) NOT NULL DEFAULT 'listed',
+        files integer,
+        matched integer,
+        release_id bigint,
+        error text,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        inspected_at timestamptz
+      )`)
+    this.schema.raw(`CREATE INDEX spots_listed_idx ON spots (posted_at DESC) WHERE status = 'listed'`)
+    this.schema.raw(`CREATE INDEX spots_unmatched_idx ON spots (inspected_at) WHERE status = 'inspected' AND matched = 0 AND release_id IS NULL`)
+    this.schema.raw(`
+      CREATE TABLE spot_files (
+        spot_id text NOT NULL,
+        idx integer NOT NULL,
+        name text NOT NULL,
+        subject text NOT NULL,
+        poster text,
+        groups text[],
+        posted_at timestamptz,
+        bytes bigint NOT NULL,
+        segments integer NOT NULL,
+        release_id bigint,
+        PRIMARY KEY (spot_id, idx)
+      )`)
+    this.schema.raw(`CREATE INDEX spot_files_release_idx ON spot_files (release_id) WHERE release_id IS NOT NULL`)
+    this.schema.raw(`
+      CREATE TABLE release_spots (
+        release_id bigint NOT NULL,
+        spot_id text NOT NULL,
+        files integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (release_id, spot_id)
+      )`)
+    this.schema.raw(`CREATE INDEX release_spots_spot_idx ON release_spots (spot_id)`)
+    this.schema.raw(`
+      CREATE TABLE release_passwords (
+        release_id bigint NOT NULL,
+        password text NOT NULL,
+        source varchar(16) NOT NULL,
+        spot_id text,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (release_id, password)
+      )`)
+    this.schema.raw(`
+      CREATE TABLE spot_state (
+        group_name text PRIMARY KEY,
+        last_article bigint NOT NULL
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS spots, spot_files, release_spots, release_passwords, spot_state`)
+  }
+}

database/migrations/1770000002900_spots_rematched.ts

@@ -0,0 +1,21 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Unmatched spots are matched again in turns (`rematched_at`), for 30 days after their inspection, also once they
+ * became a release of their own: the header pass may build their releases later (a spot's history before the
+ * catalog's), and then the spot moves to that release (LOG.md 2026-10-13 (45)).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS rematched_at timestamptz`)
+    // Inspected before the inspector set it (2026-10-13).
+    this.schema.raw(`UPDATE spots SET matched = 0 WHERE status = 'inspected' AND matched IS NULL`)
+    this.schema.raw(`DROP INDEX IF EXISTS spots_unmatched_idx`)
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS spots_unmatched_idx ON spots (coalesce(rematched_at, inspected_at)) WHERE status = 'inspected' AND matched = 0`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX IF EXISTS spots_unmatched_idx`)
+    this.schema.raw(`ALTER TABLE spots DROP COLUMN IF EXISTS rematched_at`)
+  }
+}

database/migrations/1770000003000_spots_nzb_only_unmatched.ts

@@ -0,0 +1,10 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** A matched spot's NZB isn't kept (its releases have their chunks): only a spot of its own needs one (2.7 GB at 9k spots). */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`UPDATE spots SET nzb = NULL WHERE matched > 0 AND nzb IS NOT NULL`)
+  }
+
+  async down() {}
+}

database/migrations/1770000003100_chunk_indexes_with_chunks.ts

@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * release_chunks' indexes with its chunks in CHUNK_TABLESPACE (the HDD), not the catalog's tablespace: a chunk is
+ * read from the HDD anyway, and the October partition's three indexes took 2.9 GB of the NVMe (its pkey never
+ * scanned; LOG.md 2026-10-14 (54)). The parent's partitioned indexes too, so new partitions put theirs there.
+ * Homelab was moved by hand first; a no-op where they're there already.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    if (!tablespace) return
+    this.defer(async (db) => {
+      const { rows } = await db.rawQuery(
+        `SELECT i.relname FROM pg_class c JOIN pg_index x ON x.indrelid = c.oid JOIN pg_class i ON i.oid = x.indexrelid
+         LEFT JOIN pg_tablespace t ON t.oid = i.reltablespace
+         WHERE c.relname LIKE 'release_chunks%' AND coalesce(t.spcname, 'pg_default') <> ?`,
+        [tablespace]
+      )
+      for (const { relname } of rows) await db.rawQuery(`ALTER INDEX ${relname} SET TABLESPACE ${tablespace}`)
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003200_rarely_read_tables_with_chunks.ts

@@ -0,0 +1,32 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * par2_files, fragments and probe_cache, with their indexes, in CHUNK_TABLESPACE (the HDD): ~20 GB of the NVMe at
+ * 42M releases, read by key only (the merger, the fragment merger, the prober). The catalog was at its
+ * CATALOG_MAX_NVME_GB cap and history paused (LOG.md 2026-10-14 (55)). Homelab was moved by hand first (3 min,
+ * with the writers stopped); a no-op where they're there already.
+ */
+const TABLES = ['par2_files', 'fragments', 'probe_cache']
+
+export default class extends BaseSchema {
+  async up() {
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    if (!tablespace) return
+    this.defer(async (db) => {
+      for (const table of TABLES) {
+        const { rows } = await db.rawQuery(
+          `SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+           WHERE (c.relname = ? OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = ?::regclass))
+             AND coalesce(t.spcname, 'pg_default') <> ?`,
+          [table, table, tablespace]
+        )
+        for (const { relname, relkind } of rows) {
+          await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${tablespace}`)
+        }
+      }
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003300_release_verdicts.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The junk judge's verdicts (app/services/release_judge.ts): per release the rule that judged it, after its
+ * probes, and the rule set's version. Search leaves judged releases out. Hidden only for now (nothing dropped):
+ * a verdict whose rule no longer matches (a password found, a spot linked) is taken back by the next round.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE release_verdicts (
+        release_id bigint PRIMARY KEY,
+        rule varchar(16) NOT NULL,
+        verdict varchar(8) NOT NULL DEFAULT 'hidden',
+        rule_version integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+    this.schema.raw(`CREATE INDEX release_verdicts_rule_idx ON release_verdicts (rule)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS release_verdicts`)
+  }
+}

database/migrations/1770000003400_fold_candidates.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Small releases the settler just settled (at most 3 files, under 1 MB), queued for the merger's sidecar fold
+ * (app/services/sidecar_folder.ts): an .nfo, .sfv or .nzb that came apart from its upload (another counter, another
+ * name) joins it. A queue, so the fold needn't scan the releases (LOG.md 2026-10-15 (57)).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE fold_candidates (
+        release_id bigint PRIMARY KEY,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS fold_candidates`)
+  }
+}

database/migrations/1770000003500_fragments_back_on_nvme.ts

@@ -0,0 +1,29 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * fragments, with its indexes, back in the catalog's tablespace (default_tablespace: nvme on homelab), out of
+ * CHUNK_TABLESPACE where 3200 put it: it isn't read by key only. The fragment merger walks its file_key index and
+ * reads each row, written and deleted all the time; on the HDD one round of its 30 s budget took 12+ minutes, and the
+ * coordinator's settling and tail planning waited behind it (LOG.md 2026-10-15 (57)). 13 GB. Homelab was moved by
+ * hand first, with the writers stopped; a no-op where it's there already, or without a default tablespace (tests).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
+      const target = setting[0].target
+      if (!target) return
+      const { rows } = await db.rawQuery(
+        `SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+         WHERE (c.relname = 'fragments' OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = 'fragments'::regclass))
+           AND coalesce(t.spcname, 'pg_default') <> ?`,
+        [target]
+      )
+      for (const { relname, relkind } of rows) {
+        await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`)
+      }
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003600_worker_cursors.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Where a worker's incremental rounds go on after a restart. The merger kept its "since" in memory only: each deploy
+ * or pause started it from 1970, a first round over every par2 set and inner stem ever found (25+ min with par2_files
+ * on the HDD) before the sidecar fold's turn (LOG.md 2026-10-15 (58)).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE worker_cursors (
+        name text PRIMARY KEY,
+        at timestamptz NOT NULL
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS worker_cursors`)
+  }
+}

database/migrations/1770000003700_par2_files_back_on_nvme.ts

@@ -0,0 +1,29 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * par2_files, with its indexes, back in the catalog's tablespace (default_tablespace: nvme on homelab), out of
+ * CHUNK_TABLESPACE where 3200 put it: the merger matches every new file hash against it and reads each set's rows, a
+ * random read per row on the HDD (~1,090 sets/min, a full round ~28 h; LOG.md 2026-10-15 (58)). Homelab rebuilt it
+ * there by hand without the complete and judged sets' rows (app/services/par2_pruner.ts; LOG.md 2026-10-16 (60)); a
+ * no-op where it's there already, or without a default tablespace (tests).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
+      const target = setting[0].target
+      if (!target) return
+      const { rows } = await db.rawQuery(
+        `SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+         WHERE (c.relname = 'par2_files' OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = 'par2_files'::regclass))
+           AND coalesce(t.spcname, 'pg_default') <> ?`,
+        [target]
+      )
+      for (const { relname, relkind } of rows) {
+        await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`)
+      }
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003800_rules_dropped_posters.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A rule set version with droppedPosters (lib/rules/rule_set.ts): the newest set plus the CelebMovie family, whose
+ * scraped celebrity pictures workers drop at the header pass from now on (LOG.md 2026-10-16 (65)). A new version, so the
+ * ranges record which ran with it. Nothing where there's no rule set yet (the default gets it) or the newest has it.
+ */
+const POSTERS = ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes']
+
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const latest = await db.from('rule_sets').orderBy('version', 'desc').first()
+      if (!latest || 'droppedPosters' in latest.rules) return
+      await db.table('rule_sets').insert({
+        version: Number(latest.version) + 1,
+        rules: JSON.stringify({ ...latest.rules, droppedPosters: POSTERS }),
+        note: 'dropped posters: the CelebMovie family (2026-10-16)',
+      })
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003900_final_completeness.ts

@@ -0,0 +1,45 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Final completeness by time window (LOG.md 2026-10-16 (67)). A release settles once quiet, but history is read
+ * newest first and in islands: one next to an unread range was "incomplete" twice as often (52.5% against 25.7%),
+ * its missing files only not read yet. An upload is posted within hours, so a release is final (its complete verdict
+ * definite) once done ranges cover its posting times and FINAL_MARGIN either side.
+ * - ranges.posted_low/high: the posting time span the workers report per batch (postedSpan: without the earliest and
+ *   latest 1%), the min and max over the range's batches.
+ * - range_blocks: runs of consecutive done ranges per group with their span (RangeBlocks rebuilds it); at_group_start
+ *   when the run starts at the group's first range (nothing older to wait for).
+ * - release_final(): whether one block holds the release's articles and reaches FINAL_MARGIN past its posting times.
+ *   Computed when asked, so no release row is written (50M updates would double the table on the NVMe).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS posted_low timestamptz, ADD COLUMN IF NOT EXISTS posted_high timestamptz`)
+    this.schema.raw(`
+      CREATE TABLE range_blocks (
+        group_id integer NOT NULL,
+        start_id bigint NOT NULL,
+        end_id bigint NOT NULL,
+        posted_low timestamptz,
+        posted_high timestamptz,
+        at_group_start boolean NOT NULL,
+        PRIMARY KEY (group_id, start_id)
+      )`)
+    // FINAL_MARGIN: 6 hours.
+    this.schema.raw(`
+      CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz)
+      RETURNS boolean LANGUAGE sql STABLE AS $$
+        SELECT EXISTS (
+          SELECT 1 FROM range_blocks b
+          WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article
+            AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours')
+            AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours')
+      $$`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`)
+    this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
+    this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS posted_low, DROP COLUMN IF EXISTS posted_high`)
+  }
+}

database/migrations/1770000004000_posters.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Poster search (LOG.md 2026-10-16 (70)): "poster ILIKE '%…%'" on releases had no index, so it walked all 50M releases
+ * newest first; a poster with few releases took 30+ minutes. posters holds each poster once (3M, many random per file
+ * of an obfuscated upload) with a trigram index; releases_poster_newest_idx gives a poster's releases newest first.
+ * Both by the poster's first 200 characters: a spam poster of 3,024 bytes is over a btree entry's 2,704. The batch store
+ * adds new posters. Homelab built the releases index by hand first (concurrently); a no-op there.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE TABLE IF NOT EXISTS posters (poster text PRIMARY KEY)`)
+    this.schema.raw(`INSERT INTO posters (poster) SELECT DISTINCT left(poster, 200) FROM releases WHERE poster IS NOT NULL ON CONFLICT DO NOTHING`)
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_poster_newest_idx ON releases (left(poster, 200), posted_at DESC NULLS LAST, id DESC)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_poster_newest_idx`)
+    this.schema.raw(`DROP TABLE IF EXISTS posters`)
+  }
+}

database/migrations/1770000004100_release_final_complete.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * release_final() with the release's complete flag: a release with every file its counter announced is final as it is
+ * (nothing it lacks can come); the time window only decides for the incomplete and those without a counter (LOG.md
+ * 2026-10-16 (70): a complete episode posted an hour before was "pending" for six hours more).
+ */
+export default class extends BaseSchema {
+  async up() {
+    // Beside the five-argument one, which the services still running call until the deploy restarts them.
+    this.schema.raw(`
+      CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz, complete boolean)
+      RETURNS boolean LANGUAGE sql STABLE AS $$
+        SELECT coalesce(complete, false) OR EXISTS (
+          SELECT 1 FROM range_blocks b
+          WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article
+            AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours')
+            AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours')
+      $$`)
+  }
+
+  async down() {}
+}