Accounts with limits: a Newznab key and a tier per user, daily API and NZB counts, invites

ACCESS=accounts (config/access.ts; open stays the default and behaves as
before): Newznab and the catalog's releases routes name a user (apikey or
the web interface's token), whose tier sets searches and NZBs per UTC day
(Newznab 500/501, the catalog 429) and how far back they see (102/403).
Counted in one upsert per use, so the last one can't be taken twice;
caps, stats and refused requests aren't counted. Signup takes an invite
(SIGNUP), which may give a tier; ace access:invite, access:tier (extends a
running tier), access:suspend. The web interface signs in and shows the
day's usage and the Newznab key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.adonisjs/server/controllers.ts

@@ -6,6 +6,7 @@
 export const controllers = {
   AccessTokens: () => import('#controllers/access_tokens_controller'),
   NewAccount: () => import('#controllers/new_account_controller'),
+  Newznab: () => import('#controllers/newznab_controller'),
   Node: () => import('#controllers/node_controller'),
   Profile: () => import('#controllers/profile_controller'),
   Releases: () => import('#controllers/releases_controller'),

.adonisjs/server/routes.d.ts

@@ -4,29 +4,39 @@ type ParamValue = string | number | bigint | boolean
 
 export type ScannedRoutes = {
   ALL: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'auth.new_account.store': { paramsTuple?: []; params?: {} }
     'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.lease': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   GET: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   HEAD: {
+    'newznab': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
     'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   POST: {
@@ -38,6 +48,7 @@ export type ScannedRoutes = {
     'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
 }
 declare module '@adonisjs/core/types/http' {

.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+build
+tmp
+.env
+.git
+edge/target

.env.example

@@ -19,6 +19,11 @@ DB_USER=usenet_indexer
 DB_PASSWORD=
 DB_DATABASE=usenet_rebuild
 
+# Access (config/access.ts): open (NEWZNAB_API_KEY or none, no limits) or accounts
+# (a Newznab key per user, tiers with daily limits; ace access:invite, access:tier, access:suspend)
+# ACCESS=open
+# SIGNUP=open|invite|closed (with accounts: invite unless set)
+
 #--------------------------------------------------------------------
 # CORS (configure allowed origins for API access)
 #--------------------------------------------------------------------

.gitea/workflows/deploy.yaml

@@ -1,9 +1,12 @@
-# Tests every push; a push to master that passes is deployed to homelab.
+# Tests every push; a push to rebuild (voyager 2) that passes is deployed to
+# homelab.
 #
 # Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
 # run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
-# which fetches the commit from this repo, builds the image, runs the
+# which passes "deploy-rebuild <sha>" on to ~/bin/deploy-usenet-rebuild
-# migrations and recreates the app services (never postgres or redis).
+# (deploy/deploy.sh): it fetches the commit from this repo, builds the
+# images, runs the migrations and recreates the app services (never
+# postgres). See deploy/README.md.
 name: Test and deploy
 
 on:
@@ -63,9 +66,29 @@ jobs:
       - name: Test
         run: node ace test
 
+  # The edge worker (edge/, Rust): its tests, and the Rust and TypeScript
+  # RangeIndexers compared on the committed captures (edge/fixtures).
+  edge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci
+      - name: Rust toolchain
+        run: |
+          command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+      - name: cargo test
+        run: cargo test --release --manifest-path edge/Cargo.toml
+      - name: Rust and TypeScript identical on the fixtures
+        run: edge/check-fixtures.sh
+
   deploy:
-    needs: test
+    needs: [test, edge]
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/rebuild'
     runs-on: ubuntu-latest
     steps:
       - name: Deploy to homelab
@@ -78,4 +101,4 @@ jobs:
           chmod 600 ~/.ssh/deploy_key
           printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
           ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
-            daan@10.206.0.175 "deploy ${{ github.sha }}"
+            daan@10.206.0.175 "deploy-rebuild ${{ github.sha }}"

.gitignore

@@ -24,3 +24,6 @@ yarn-error.log
 
 # Platform specific
 .DS_Store
+
+# Generated by the dev tooling
+.adonisjs/server/routes.json

Dockerfile

@@ -0,0 +1,43 @@
+# syntax=docker/dockerfile:1
+#
+# Voyager 2's images (deploy/compose.yaml):
+#   --target app   the AdonisJS app: web server, coordinator, migrations, ace commands
+#   --target edge  the Rust edge worker (edge/, bin edge-node)
+
+# Node is pinned as in CI (.nvmrc): on 24.21, jsonschema 1.5.0 (used by Ace to
+# validate commands) throws "Invalid URL" on $ref, so no ace command starts.
+ARG NODE_VERSION=24.16
+
+# The app, compiled, with production dependencies only.
+FROM node:${NODE_VERSION}-bookworm AS app-build
+WORKDIR /src
+COPY package.json package-lock.json ./
+RUN npm ci
+COPY . .
+RUN node ace build \
+ && cd build \
+ && npm ci --omit=dev
+
+FROM node:${NODE_VERSION}-bookworm-slim AS app
+ENV NODE_ENV=production
+WORKDIR /app
+COPY --from=app-build /src/build ./
+USER node
+# Every service picks its own command, e.g. `node ace coordinator:run`.
+CMD ["node", "ace", "list"]
+
+# The Rust worker. Registry and target dir are build caches, so a rebuild only compiles what changed.
+FROM rust:1-bookworm AS edge-build
+WORKDIR /src
+COPY edge ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/src/target \
+    cargo build --release --locked --bin edge-node --bin edge-bench \
+ && cp target/release/edge-node target/release/edge-bench /usr/local/bin/
+
+FROM debian:bookworm-slim AS edge
+COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node
+# edge-bench replays the corpus for the grouping sweeps (deploy/corpus_sweep.py).
+COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
+USER 1000:1000
+ENTRYPOINT ["/usr/local/bin/edge-node"]

REBUILD.md

@@ -1,4 +1,9 @@
-# Rebuild: header-first indexer (spec, draft for approval)
+# Voyager 2: header-first indexer (concept)
+
+The concept for what runs (since 2026-10-11; it started as the spec of a
+rebuild, then a draft for approval). Voyager 1's CONCEPT.md and its
+decisions D1–D42 are superseded; its domain model still holds, and its
+features not yet here are the phase 2 backlog below.
 
 Branch `rebuild`. Phase 1 finds releases from headers and makes their NZBs.
 Everything learned from articles (names from yEnc headers, par2, archive
@@ -54,8 +59,21 @@ most workers on batcave, a few on poweredge-kantoor with
 `edge:node --max-mbit` (e.g. 50 of its 160 Mbit/s ≈ 100–125 k articles/s;
 the cap is shared by the node's processes).
 
+Also built (2026-10-09): the live tail (new articles planned every 2
+minutes from one LIST ACTIVE per name prefix, at priority −1: leased and
+fetched before history); the catalog API, Newznab (`/api`) and the web
+interface, from voyager 1 (the current build), with categories from the
+name; voyager 1's 29 learned garbage signatures (`rules:signatures`,
+`database/data/voyager1_signatures.tsv`).
+
+Deployment (2026-10-10): `Dockerfile` (targets app and edge),
+`deploy/compose.yaml` (project usenet-rebuild on homelab), `deploy/deploy.sh`;
+a push to `rebuild` that passes CI is deployed. See `deploy/README.md`.
+
 Not built yet: re-running ranges for a retracted rule (the counts and rule
-versions are recorded), deployment (Dockerfile, compose), phase 2.
+versions are recorded), decision f, phase
+2 (inspection of the probe queue: yEnc names, encryption, archives, par2;
+passwords; spots).
 
 ## Target
 
@@ -101,7 +119,7 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
 2. **Edge worker** (node, TypeScript reusing `lib/`). Measured (`edge:bench`,
    2026-10-09, after two rounds of work): ~200–215 k headers/s per core on
    recent a.b.boneless, ~320–340 k on a 2013 a.b.ath range, ~0.7 GB peak
-   per process. **Rust** (`tools/edge-bench-rs`, the same RangeIndexer path
+   per process. **Rust** (`edge/`, the same RangeIndexer path
    over the same captures; `check.ts` finds every output identical, the
    decoded payloads included): 3.5–3.8× less CPU (boneless 2.9 vs 11.1
    CPU-s per 2 M headers, ath 2.15 vs 7.5) and a third of the memory
@@ -111,6 +129,10 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
    instead of ~6–7; the speed is still set by the NNTP connections. The
    cost: a second implementation of lib/headers, lib/release, lib/rules and
    the chunk format, kept identical (check.ts) for as long as both exist.
+   Findings, how to run both and how to compare their outputs:
+   `edge/README.md`. Since 2026-10-09 the Rust worker
+   (`edge-node`, same crate) is the one running on homelab: ~8× less CPU
+   per header and 20× less memory than `edge:node` in production.
    Per lease: compressed chunks of 10–50 k lines, in order →
    parse subjects → skip crossposts outside their owner group (of the
    indexed groups in the article's Xref, the one with the lowest priority
@@ -186,11 +208,109 @@ of other groups whose counts show crossposts skipped for it (part 5). Start:
 the 1000 largest `alt.binaries.*` groups by LIST ACTIVE (~266 k ranges of
 2 M articles; boneless alone 80 k).
 
-## Not in phase 1
+## Corpus
 
-Inspection and archive walks, par2/SFV naming, spots, NZB imports, the
+The grouping sweeps' dataset (2026-10-12; LOG.md 2026-10-12 (12)): the XOVER
-after-probe garbage deletes, live indexing (the current indexer keeps
+chunks of ranges marked `capture`, kept on the HDD in
-running and does those meanwhile).
+`corpus/<group>/<start>-<end>/` (edge-bench's layout, `range.json` once
+complete), up to 300 GB. The prefetcher stores them as it fetches and reads
+them from there afterwards, so a reset re-indexes them without the news
+server. `deploy/corpus_sweep.py` marks more ranges each sweep (`capture`,
+spread over groups and eras, the top-100 groups first), replays every stored
+range with the deployed build's edge-bench (`replay`, ~seconds a range) and
+reports suspects and the change since the previous build (`report`). A rule
+change is judged on the whole corpus before a reset.
+
+## Phase 2
+
+Built: the prober (2026-10-11; `probe:run`, LOG.md 2026-10-11 (2)). It
+reads the probe queue's first articles (`lone`, `par2`, `archive`) and
+stores per release whether it's passworded and its inner name, which search,
+the web interface and Newznab use.
+
+Built: the par2 merge (2026-10-12; `releases:merge`, the `merger` service,
+LOG.md 2026-10-12 (13)). The prober stores per probed file start the MD5 of
+its first 16 KiB (`file_hashes`) and per par2 body its set's file list
+(`par2_files`, `par2_sets`), which holds the same hash per file whatever the
+file was renamed to. Every minute the merger merges the releases of one
+group, posted within 48 hours of a release holding the set's par2, whose
+files the set describes, into the one with the most files (chunks,
+candidates, inspections, counts; the par2's shared name when none was
+readable); `release_merges` keeps what went where. Obfuscated uploads whose
+files were each a release of their own (`lone` candidates) become one.
+
+**Backlog: voyager 1's features not in voyager 2** (CONCEPT.md decisions;
+voyager 1's data exported to `/mnt/bulk/usenet-rebuild/v1-export/` on
+homelab: `voyager1-tables.dump` with 2.76 M spots, upload signatures,
+signature probes, dropped releases and groups; `voyager1-release-passwords.csv`
+with 68,032 releases with a password):
+
+| Feature | Voyager 1 | In voyager 2 |
+|---|---|---|
+| Spotnet spots | D29 | the spotter and the spot inspector (below); new spots only, the export's not imported |
+| Posted NZBs read into releases (also NZBs inside archives) | D26, D28 | not yet |
+| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
+| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
+| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
+| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed |
+| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
+| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
+
+Live indexing (the tail) runs since 2026-10-09.
+
+**Junk** (voyager 1's second filter, D42). After the header pass's rules
+(garbage signatures, executables, per-article and obfuscated subjects), the
+judge (`releases:judge`, service `judge`) judges releases hourly by what the
+prober, the merges and the spots found, in this order (the first rule
+wins): *dead* (every probed first article gone), *fake* (a film, episode
+or music name whose content is a program: archive entries all
+executables, or the file itself one by name or by its first bytes, sniffed
+`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
+*locked* (passworded, no password from a spot, its NZB or the title),
+*unusable* (random names, no inner name, no par2 or inner-name merge, no
+spot, a day after settling). Verdicts (`release_verdicts`, with the rule
+set's version) hide releases from search (`judged` shows them); a verdict
+whose rule no longer matches is taken back. Nothing is dropped yet.
+
+**Sidecars** (the merger's third pass). An upload's .nfo, .sfv or .nzb
+that came apart from it (another name, another counter: "[00/80] … .nzb"
+beside "[01/94] … .part01.rar") was a release of its own. The settler
+queues small settled releases (≤ 3 files, < 1 MB) in `fold_candidates`;
+one whose files are all sidecars joins the poster's bigger release in the
+same group posted within 10 minutes: the one sharing a word of 4+ letters
+with its subject, else the one under the same counter total, else none
+(`release_merges.via` 'sidecar').
+
+**Spots** (after voyager 1's D29; `lib/spotnet` ported as is). Two
+processes, one NNTP connection each:
+
+- **The spotter** (`spots:read`, service `spotter`) reads free.pt's
+  overviews after the last article it saw (`spot_state`; the first time the
+  last `SPOT_BACKFILL`, 20,000, ~3 months), every `SPOT_EVERY_SECONDS`
+  (120). For each spot it reads the full header: the signature is checked
+  there as Spotweb does (Spotnet's keys 2–4; self-signed key 7 by hashcash),
+  and the X-XML gives the description, the website and the NZB's articles.
+  A password the title (`{{pw}}`) or the description names
+  (`passwordFromSpot`) is kept with the spot, with its source. A
+  delete/dispose message marks its target `removed` (a personal one only
+  from the same spotter within 5 days) and takes its links and passwords
+  back from the releases.
+- **The spot inspector** (`spots:inspect`, service `spot-inspector`) looks
+  inside each listed spot, newest first: it reads the compressed NZB (≤
+  `SPOT_MAX_NZB_SEGMENTS` articles in alt.binaries.ftd), lists its files
+  in `spot_files` (named as the header pass names them, `parseSubject`),
+  and matches each file to the release the header pass built for it: the
+  same release identity (`releaseIdentity` over the file's subject and
+  poster) in one of its groups, the release posted nearest the file within
+  48 h. Matched releases are linked in `release_spots`. Placeholder NZBs
+  (`dummy`), gone articles (`nzb-gone`) and unreadable ones are noted on the
+  spot. Unmatched spots are matched again every hour; after
+  `SPOT_STANDALONE_AFTER_HOURS` (24) a spot becomes a release of its own
+  (source `spot`), whose NZB is the spot's (kept gzipped in `spots.nzb`).
+- **Passwords are kept apart** in `release_passwords` (release, password,
+  source: `nzb` for the NZB's `<meta type="password">`, else `spot-title` or
+  `spot-text`, and the spot), not on the catalog row; the catalog only says
+  `passworded`.
 
 ## Decisions
 
@@ -201,3 +321,8 @@ c. Group list: editable, owner by priority number (above). **Accepted.**
 d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
 e. Headers through the coordinator's relay, with prefetch (part 1).
    **Accepted.**
+f. Random-named releases (no file with a readable name) keep no chunk in
+   the header pass: their file keys and probe candidates only, until
+   phase 2 names them (bodies) and fetches their segments again. The full
+   history is ~3 TB with them, ~1.2–1.4 TB without (725 M of the 1.25 B
+   kept files; LOG.md 2026-10-09). **Accepted** 2026-10-09, not built yet.

adonisrc.ts

@@ -53,6 +53,7 @@ export default defineConfig({
     () => import('@adonisjs/cors/cors_provider'),
     () => import('@adonisjs/auth/auth_provider'),
     () => import('#providers/api_provider'),
+    () => import('@adonisjs/static/static_provider')
   ],
 
   /*
@@ -103,7 +104,10 @@ export default defineConfig({
   | the production build.
   |
   */
-  metaFiles: [],
+  metaFiles: [{
+    pattern: 'public/**',
+    reloadServer: false,
+  }],
 
   hooks: {
     init: [

app/controllers/new_account_controller.ts

@@ -1,13 +1,36 @@
 import User from '#models/user'
+import db from '@adonisjs/lucid/services/db'
 import { signupValidator } from '#validators/user'
 import type { HttpContext } from '@adonisjs/core/http'
 import UserTransformer from '#transformers/user_transformer'
+import AccessService from '#services/access_service'
+
+class InviteGone extends Error {}
 
 export default class NewAccountController {
-  async store({ request, serialize }: HttpContext) {
+  async store({ request, response, serialize }: HttpContext) {
-    const { fullName, email, password } = await request.validateUsing(signupValidator)
+    const { fullName, email, password, invite } = await request.validateUsing(signupValidator)
+    if (AccessService.signup === 'closed') return response.forbidden({ error: 'Signup is closed' })
+    if (AccessService.signup === 'invite' && !invite) return response.forbidden({ error: 'An invite code is needed to sign up' })
 
-    const user = await User.create({ fullName, email, password })
+    // The user and their invite together: an invite that's gone rolls the signup back.
+    const user = await db
+      .transaction(async (trx) => {
+        const created = await User.create({ fullName, email, password }, { client: trx })
+        if (AccessService.signup !== 'invite') return created
+        const gives = await AccessService.redeem(invite!, created.id, trx)
+        if (!gives) throw new InviteGone()
+        if (gives.tier) {
+          AccessService.grantTier(created, gives.tier, gives.tier_days)
+          await created.useTransaction(trx).save()
+        }
+        return created
+      })
+      .catch((error) => {
+        if (error instanceof InviteGone) return null
+        throw error
+      })
+    if (!user) return response.forbidden({ error: 'Unknown or used invite code' })
     const token = await User.accessTokens.create(user)
 
     return serialize({

app/controllers/newznab_controller.ts

@@ -0,0 +1,141 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import env from '#start/env'
+import NzbService from '#services/nzb_service'
+import SearchService from '#services/search_service'
+import AccessService, { AccessDenied } from '#services/access_service'
+import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
+
+const LIMITS = { max: 100, default: 50 }
+
+/** Search functions and the category each defaults to when `cat` is absent. */
+const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
+  'search': undefined,
+  'tvsearch': 5000,
+  'tv-search': 5000,
+  'movie': 2000,
+  'movie-search': 2000,
+  'music': 3000,
+  'audio': 3000,
+  'book': 7000,
+}
+
+/** The configured key, if any. */
+function configuredApiKey(): string | undefined {
+  const value: any = env.get('NEWZNAB_API_KEY')
+  const key = typeof value === 'string' ? value : value?.release()
+  return key || undefined
+}
+
+function int(value: unknown): number | undefined {
+  return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined
+}
+
+/**
+ * Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1):
+ * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
+ * Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
+ */
+export default class NewznabController {
+  async handle(ctx: HttpContext) {
+    const xml = (body: string, type = 'application/xml') => {
+      ctx.response.header('Content-Type', `${type}; charset=utf-8`)
+      return ctx.response.send(body)
+    }
+    try {
+      return await this.answer(ctx, xml)
+    } catch (error) {
+      if (error instanceof AccessDenied) return xml(errorXml(error.newznabCode, error.message))
+      throw error
+    }
+  }
+
+  private async answer(ctx: HttpContext, xml: (body: string, type?: string) => void) {
+    const { request, response } = ctx
+    const qs = request.qs()
+    const t = String(qs.t ?? '')
+
+    // With accounts the key is a user's (AccessService); else the one configured, if any.
+    const grant = await AccessService.authorize(ctx, qs.apikey)
+    const apiKey = grant ? grant.user.apiKey : configuredApiKey()
+    if (!grant && apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
+    const server = this.server(request, apiKey)
+    if (t === 'caps') return xml(capsXml(server, LIMITS, { available: AccessService.accounts && AccessService.signup !== 'closed', open: AccessService.accounts && AccessService.signup === 'open' }))
+
+    if (t === 'get') {
+      const id = int(qs.id)
+      if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
+      // Unknown ids and releases past the tier's retention aren't counted.
+      if (!(await AccessService.checkRelease(grant, id))) return xml(errorXml(300, 'No such item'))
+      await AccessService.charge(grant, 'grab')
+      const nzb = await NzbService.build(id)
+      if (!nzb) return xml(errorXml(300, 'No such item'))
+      const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
+      response.header('Content-Disposition', `attachment; filename="${filename}"`)
+      return xml(nzb.xml, 'application/x-nzb')
+    }
+
+    if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
+    await AccessService.charge(grant, 'api')
+
+    const requested = String(qs.cat ?? '')
+      .split(',')
+      .map((c) => int(c.trim()))
+      .filter((c): c is number => c !== undefined)
+    const fallback = SEARCH_FUNCTIONS[t]
+    const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : [])
+    const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
+    const offset = int(qs.offset) ?? 0
+
+    // ID-only lookups (tvdbid, imdbid, …) aren't supported: an empty result rather than everything.
+    const q = typeof qs.q === 'string' ? qs.q.trim() : ''
+    const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
+    const result = idOnly
+      ? null
+      : await SearchService.search({
+          q,
+          categories,
+          readable: true,
+          notIncomplete: true,
+          maxAgeDays: AccessService.maxAgeDays(grant, int(qs.maxage)),
+          season: int(qs.season),
+          episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
+          sort: 'newest',
+          perPage: limit,
+          offset,
+        })
+
+    const items = (result?.releases ?? []).map((r) => {
+      const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
+      return {
+        id: r.id,
+        // An obfuscated release by what the prober found inside, without its extension (what clients parse).
+        title: r.readable || !r.innerName ? r.name : r.innerName.replace(/\.[a-z0-9]{2,4}$/i, ''),
+        category: r.category ?? 8010,
+        size: r.size,
+        files: r.fileCount,
+        postedAt: r.postedAt ? new Date(r.postedAt) : null,
+        poster: r.poster,
+        groups: r.groups ?? [],
+        // From the prober (phase 2): encrypted archive entries or headers.
+        password: r.passworded === true,
+        season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
+        episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
+      }
+    })
+    // What's left of the day, for clients that show it (newznab:apilimits).
+    const limits = grant && {
+      apiCurrent: grant.usage.apiHits,
+      apiMax: grant.limits.apiHits,
+      grabCurrent: grant.usage.grabs,
+      grabMax: grant.limits.grabs,
+    }
+    return xml(resultsXml(server, items, offset, result?.total ?? 0, limits || undefined), 'application/rss+xml')
+  }
+
+  /** Download links carry the caller's key. */
+  private server(request: HttpContext['request'], apiKey: string | undefined): NewznabServer {
+    const proto = request.header('x-forwarded-proto') ?? 'http'
+    const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
+    return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey }
+  }
+}

app/controllers/node_controller.ts

@@ -6,8 +6,12 @@ import BatchStore, { StaleBatch } from '#services/batch_store'
 import Spool from '#services/spool'
 import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
 
-/** How long the stream waits for a chunk the prefetcher hasn't spooled yet. */
+/**
-const CHUNK_WAIT_MS = 15 * 60 * 1000
+ * How long the stream waits for a chunk the prefetcher hasn't spooled yet. Long: a chunk that keeps
+ * coming in isn't given up (only a stalled one is), and some groups' chunks take 20+ minutes
+ * (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
+ */
+const CHUNK_WAIT_MS = 60 * 60 * 1000
 /** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
 const KEEPALIVE_MS = 20_000
 

app/controllers/profile_controller.ts

@@ -1,8 +1,19 @@
 import UserTransformer from '#transformers/user_transformer'
 import type { HttpContext } from '@adonisjs/core/http'
+import AccessService from '#services/access_service'
 
 export default class ProfileController {
+  /** The user, their tier, Newznab key and what they used today. */
   async show({ auth, serialize }: HttpContext) {
-    return serialize(UserTransformer.transform(auth.use('api').getUserOrFail()))
+    const user = auth.use('api').getUserOrFail()
+    return serialize({ user: UserTransformer.transform(user), access: await AccessService.summary(user) })
+  }
+
+  /** POST /api/v1/account/api-key: a new Newznab key (the old one stops working). */
+  async resetApiKey({ auth }: HttpContext) {
+    const user = auth.use('api').getUserOrFail()
+    user.apiKey = AccessService.newKey()
+    await user.save()
+    return { apiKey: user.apiKey }
   }
 }

app/controllers/releases_controller.ts

@@ -1,27 +1,77 @@
 import type { HttpContext } from '@adonisjs/core/http'
-import db from '@adonisjs/lucid/services/db'
 import NzbService from '#services/nzb_service'
+import AccessService from '#services/access_service'
+import SearchService, { type SortOrder } from '#services/search_service'
+import { expandCategories } from '#lib/newznab/index'
 
-/** The catalog (REBUILD.md, part 9): a name search and a release's NZB. */
+const flag = (value: unknown) => value === '1' || value === 'true' || value === true
+
+/** A size in MB from the query string, as bytes. */
+function megabytes(value: unknown): number | undefined {
+  const mb = Number(value)
+  return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined
+}
+
+/** The catalog for the web interface (REBUILD.md, part 9). */
 export default class ReleasesController {
-  /** Releases whose name has every word of `q` (dots, dashes, underscores and spaces alike), newest first. */
+  /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
-  async index({ request }: HttpContext) {
+  async index(ctx: HttpContext) {
-    const q = String(request.input('q', '')).trim()
+    const qs = ctx.request.qs()
-    const limit = Math.min(Number(request.input('limit', 50)) || 50, 200)
+    const grant = await AccessService.authorize(ctx, qs.apikey)
-    const query = db
+    await AccessService.charge(grant, 'api')
-      .from('releases')
+    const categories = String(qs.cat ?? '')
-      .select('id', 'name', 'poster', 'groups', 'posted_at', 'files', 'file_total', 'bytes', 'complete', 'readable')
+      .split(',')
-      .orderBy('posted_at', 'desc')
+      .map((c) => Number(c.trim()))
-      .limit(limit)
+      .filter((c) => Number.isInteger(c) && c > 0)
-    for (const word of q.split(/[\s._-]+/).filter(Boolean)) query.whereILike('name', `%${word.replace(/[\\%_]/g, '\\$&')}%`)
+    return SearchService.search({
-    return { releases: await query }
+      maxAgeDays: AccessService.maxAgeDays(grant),
+      q: typeof qs.q === 'string' ? qs.q : undefined,
+      categories: categories.length ? expandCategories(categories) : undefined,
+      readable: flag(qs.readable),
+      complete: flag(qs.complete),
+      unlocked: flag(qs.unlocked),
+      source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
+      poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
+      minSize: megabytes(qs.min_size),
+      maxSize: megabytes(qs.max_size),
+      sort: qs.sort as SortOrder | undefined,
+      page: Number(qs.page) || 1,
+      perPage: Number(qs.per_page) || 50,
+      // The page shows "1,000+"; counting further made filter clicks slow.
+      countLimit: 1000,
+    })
   }
 
-  async nzb({ params, response }: HttpContext) {
+  /** GET /api/v1/releases/:id, with its files */
+  async show(ctx: HttpContext) {
+    const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
+    if (!(await AccessService.checkRelease(grant, Number(ctx.params.id)))) return ctx.response.notFound({ error: 'Release not found' })
+    await AccessService.charge(grant, 'api')
+    const release = await SearchService.release(Number(ctx.params.id))
+    return release ?? ctx.response.notFound({ error: 'Release not found' })
+  }
+
+  /** GET /api/v1/releases/:id/nzb (?apikey= for a plain link, as the web interface's) */
+  async nzb(ctx: HttpContext) {
+    const { params, response } = ctx
+    const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
+    if (!(await AccessService.checkRelease(grant, Number(params.id)))) return response.notFound({ error: 'Release not found' })
+    await AccessService.charge(grant, 'grab')
     const nzb = await NzbService.build(Number(params.id))
-    if (!nzb) return response.notFound({ error: 'No such release' })
+    if (!nzb) return response.notFound({ error: 'Release not found' })
+    const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
     response.header('Content-Type', 'application/x-nzb')
-    response.header('Content-Disposition', `attachment; filename="${nzb.name.replace(/["\\/]/g, '_')}.nzb"`)
+    response.header('Content-Disposition', `attachment; filename="${filename}"`)
-    return nzb.xml
+    return response.send(nzb.xml)
+  }
+
+  /** GET /api/v1/access: whether the web interface needs an account, and how one is made. */
+  async access() {
+    return { mode: AccessService.mode, signup: AccessService.signup }
+  }
+
+  /** GET /api/v1/stats (open: what the index holds, not what's in it) */
+  async stats() {
+    return SearchService.stats()
   }
 }

app/exceptions/handler.ts

@@ -8,6 +8,9 @@ export default class HttpExceptionHandler extends ExceptionHandler {
    */
   protected debug = !app.inProduction
 
+  /** A refused request (AccessService: no key, a limit reached) is an answer, not a fault. */
+  protected ignoreCodes = ['E_ACCESS_DENIED']
+
   /**
    * The method is used for handling errors and returning
    * response to the client

app/models/user.ts

@@ -1,4 +1,6 @@
+import { randomBytes } from 'node:crypto'
 import { UserSchema } from '#database/schema'
+import { beforeCreate } from '@adonisjs/lucid/orm'
 import hash from '@adonisjs/core/services/hash'
 import { compose } from '@adonisjs/core/helpers'
 import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
@@ -8,6 +10,12 @@ export default class User extends compose(UserSchema, withAuthFinder(hash)) {
   static accessTokens = DbAccessTokensProvider.forModel(User)
   declare currentAccessToken?: AccessToken
 
+  /** The Newznab key (AccessService), known on the model right away rather than from the column's default. */
+  @beforeCreate()
+  static assignApiKey(user: User) {
+    user.apiKey ??= randomBytes(16).toString('hex')
+  }
+
   get initials() {
     const [first, last] = this.fullName ? this.fullName.split(' ') : this.email.split('@')
     if (first && last) {

app/services/access_service.ts

@@ -0,0 +1,171 @@
+import { randomBytes } from 'node:crypto'
+import { DateTime } from 'luxon'
+import db from '@adonisjs/lucid/services/db'
+import { Exception } from '@adonisjs/core/exceptions'
+import type { HttpContext } from '@adonisjs/core/http'
+import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
+import User from '#models/user'
+import { ACCESS, SIGNUP, TIERS, type TierLimits } from '#config/access'
+
+export type Use = 'api' | 'grab'
+
+/** Newznab's error codes for access (the catalog answers with the HTTP status). */
+const STATUS = { 100: 401, 101: 403, 102: 403, 500: 429, 501: 429 } as const
+
+/** Why a request is refused; the catalog's answer (Newznab catches it for its own). */
+export class AccessDenied extends Exception {
+  constructor(
+    readonly newznabCode: keyof typeof STATUS,
+    message: string
+  ) {
+    super(message, { status: STATUS[newznabCode], code: 'E_ACCESS_DENIED' })
+  }
+
+  async handle(error: this, ctx: HttpContext) {
+    ctx.response.status(error.status).send({ error: error.message, code: error.newznabCode })
+  }
+}
+
+export interface Usage {
+  apiHits: number
+  grabs: number
+}
+
+/** A caller let in: who, their tier as it is now, and what they used today. */
+export interface Grant {
+  user: User
+  tier: string
+  limits: TierLimits
+  usage: Usage
+}
+
+const COLUMNS: Record<Use, string> = { api: 'api_hits', grab: 'grabs' }
+const TODAY = `(now() AT TIME ZONE 'UTC')::date`
+const DAY_MS = 86_400_000
+
+/** Accounts, tiers and their daily limits (config/access.ts). */
+export default class AccessService {
+  /** From config/access.ts; tests switch it. */
+  static mode: 'open' | 'accounts' = ACCESS
+  static signup: 'open' | 'invite' | 'closed' = SIGNUP
+
+  static get accounts() {
+    return this.mode === 'accounts'
+  }
+
+  static newKey() {
+    return randomBytes(16).toString('hex')
+  }
+
+  /** The user's tier as it is now: one that ran out (or isn't configured) is free. */
+  static tierOf(user: User): { tier: string; limits: TierLimits } {
+    const expired = user.tierUntil !== null && user.tierUntil.toMillis() <= Date.now()
+    const tier = !expired && TIERS[user.tier] ? user.tier : 'free'
+    return { tier, limits: TIERS[tier] }
+  }
+
+  /** The user a request names: the web interface's token, else an apikey (Newznab, the catalog's NZB links). */
+  static async caller(ctx: HttpContext, apikey?: unknown): Promise<User | null> {
+    const guard = ctx.auth.use('api')
+    if (guard.isAuthenticated || (await guard.check())) return guard.user!
+    if (typeof apikey === 'string' && /^[0-9a-f]{32}$/.test(apikey)) return User.findBy('apiKey', apikey)
+    return null
+  }
+
+  /** Lets a request in (counting nothing yet); null when access is open. */
+  static async authorize(ctx: HttpContext, apikey?: unknown): Promise<Grant | null> {
+    if (!this.accounts) return null
+    const user = await this.caller(ctx, apikey)
+    if (!user) throw new AccessDenied(100, 'Incorrect user credentials')
+    if (user.suspended) throw new AccessDenied(101, 'Account suspended')
+    return { user, ...this.tierOf(user), usage: await this.usage(user.id) }
+  }
+
+  /** Counts one use against the grant's tier, or refuses it once the day's limit is reached. */
+  static async charge(grant: Grant | null, use: Use) {
+    if (!grant) return
+    const usage = await this.count(grant.user.id, use, use === 'api' ? grant.limits.apiHits : grant.limits.grabs)
+    if (!usage) throw use === 'api' ? new AccessDenied(500, 'Request limit reached') : new AccessDenied(501, 'Download limit reached')
+    grant.usage = usage
+  }
+
+  /**
+   * Adds one use to today's row unless the limit is reached (null then): one
+   * statement, so concurrent requests can't both take the last one.
+   */
+  static async count(userId: number, use: Use, limit: number | null): Promise<Usage | null> {
+    if (limit !== null && limit <= 0) return null
+    const column = COLUMNS[use]
+    const { rows } = await db.rawQuery(
+      `INSERT INTO usage (user_id, day, ${column}) VALUES (?, ${TODAY}, 1)
+       ON CONFLICT (user_id, day) DO UPDATE SET ${column} = usage.${column} + 1
+       ${limit === null ? '' : `WHERE usage.${column} < ?`}
+       RETURNING api_hits, grabs`,
+      limit === null ? [userId] : [userId, limit]
+    )
+    return rows[0] ? { apiHits: rows[0].api_hits, grabs: rows[0].grabs } : null
+  }
+
+  static async usage(userId: number): Promise<Usage> {
+    const { rows } = await db.rawQuery(`SELECT api_hits, grabs FROM usage WHERE user_id = ? AND day = ${TODAY}`, [userId])
+    return { apiHits: rows[0]?.api_hits ?? 0, grabs: rows[0]?.grabs ?? 0 }
+  }
+
+  /** A search's age bound: the asked one within the tier's retention. */
+  static maxAgeDays(grant: Grant | null, requested?: number): number | undefined {
+    const retention = grant?.limits.retentionDays
+    if (!retention) return requested
+    return requested ? Math.min(requested, retention) : retention
+  }
+
+  /** Refuses a release posted before the tier's retention (one without a date is let through). */
+  static checkPosted(grant: Grant | null, postedAt: Date | string | null) {
+    const retention = grant?.limits.retentionDays
+    if (!retention || !postedAt) return
+    if (new Date(postedAt).getTime() < Date.now() - retention * DAY_MS) {
+      throw new AccessDenied(102, `Older than your tier's retention (${retention} days)`)
+    }
+  }
+
+  /** Whether a release exists, refusing it when it's older than the tier sees; asked before an NZB is counted. */
+  static async checkRelease(grant: Grant | null, id: number): Promise<boolean> {
+    const release = await db.from('releases').where('id', id).select('posted_at').first()
+    if (!release) return false
+    this.checkPosted(grant, release.posted_at)
+    return true
+  }
+
+  /** Takes an unused invite for a new user: what it gives, or null when it's unknown or used. */
+  static async redeem(code: string, userId: number, trx: TransactionClientContract) {
+    const { rows } = await trx.rawQuery(
+      `UPDATE invites SET used_by = ?, used_at = now() WHERE code = ? AND used_by IS NULL AND used_at IS NULL RETURNING tier, tier_days`,
+      [userId, code]
+    )
+    return (rows[0] as { tier: string | null; tier_days: number | null } | undefined) ?? null
+  }
+
+  /**
+   * Gives a user a tier for `days` (null: without end). The same tier still
+   * running is extended from where it ends, so a renewal paid early loses nothing.
+   */
+  static grantTier(user: User, tier: string, days: number | null) {
+    if (!TIERS[tier]) throw new Error(`No such tier: ${tier} (config/access.ts has ${Object.keys(TIERS).join(', ')})`)
+    const now = DateTime.utc()
+    const running = user.tier === tier && user.tierUntil && user.tierUntil > now ? user.tierUntil : now
+    user.tier = tier
+    user.tierUntil = days === null ? null : running.plus({ days })
+  }
+
+  /** What a profile shows of the account's access. */
+  static async summary(user: User) {
+    const { tier, limits } = this.tierOf(user)
+    return {
+      mode: this.mode,
+      tier,
+      tierUntil: tier === user.tier ? user.tierUntil : null,
+      apiKey: user.apiKey,
+      limits,
+      usage: await this.usage(user.id),
+    }
+  }
+}

app/services/batch_store.ts

@@ -1,7 +1,8 @@
 import db from '@adonisjs/lucid/services/db'
 import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
-import type { EdgeBatch, ProbeCandidate, ReleaseChunk } from '#lib/edge/index'
+import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
 import ChunkPartitions from '#services/chunk_partitions'
+import { decodeChunk } from '#lib/chunks/index'
 
 export class StaleBatch extends Error {}
 
@@ -23,7 +24,11 @@ class BatchStore {
   async store(rangeId: number, nodeId: number, batch: EdgeBatch): Promise<StoredBatch> {
     await ChunkPartitions.ensure()
     return db.transaction(async (trx) => {
-      const range = await trx.from('ranges').where('id', rangeId).forUpdate().first()
+      // One batch of a range at a time, without holding the range's row for the whole transaction: the
+      // prefetcher's and heartbeats' updates of it waited on it (0.5 s on average, NNTP connections idle;
+      // LOG.md 2026-10-09 (13)). The resume point is checked again when it moves, at the end.
+      await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [rangeId])
+      const range = await trx.from('ranges').where('id', rangeId).first()
       if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
         throw new StaleBatch(`Range ${rangeId} is not leased to this node`)
       if (Number(range.next_id) !== batch.from)
@@ -73,29 +78,88 @@ class BatchStore {
         const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
           range_id: rangeId,
           file_key: f.fileKey,
-          article_number: Math.min(...f.segments.map((s) => s[3])),
+          // Not Math.min(...): a big file's fragment has more segments than the call stack takes arguments.
+          article_number: f.segments.reduce((min, s) => (s[3] < min ? s[3] : min), Infinity),
           segments: f.segments.length,
           fragment: JSON.stringify(f),
         }))
         await trx.insertQuery().table('fragments').multiInsert(rows)
       }
 
-      await trx
+      const moved = await trx
         .from('ranges')
         .where('id', rangeId)
+        .where('status', 'leased')
+        .where('node_id', nodeId)
+        .where('next_id', batch.from)
         .update({
           next_id: batch.next,
           headers: db.raw('headers + ?', [batch.headers]),
+          // The range's posting time span (release_final): the batches' trimmed spans, min and max (least and greatest skip nulls).
+          posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [batch.postedLow ?? null]),
+          posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [batch.postedHigh ?? null]),
           heartbeat_at: new Date(),
         })
+      if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1)) throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
       return result
     })
   }
 
+  /**
+   * An upload spread over groups (FTD: par2 and part1 to a.b.music.misc,
+   * part2 to a.b.sounds.music, each article in one group only; LOG.md
+   * 2026-10-11 (9)): a chunk without a release in its own group joins one of
+   * its key in another group whose window is within UPLOAD_GAP_MS, when none
+   * of its file names are in that release yet. A post of the same files to
+   * another group (other message-ids, the same names) stays a release of its
+   * own, or the NZB would hold every file twice. Reads the candidate's file
+   * names from its chunks; candidates are few.
+   */
+  private async joinAcrossGroups(
+    trx: TransactionClientContract,
+    part: ReleaseChunk[],
+    windows: { idx: number; key: string; first: number | null | undefined; last: number | null | undefined }[],
+    groupId: number,
+    joined: Map<number, number>
+  ) {
+    const open = windows.filter((w) => !joined.has(w.idx) && w.first)
+    if (open.length === 0) return
+    const { rows: candidates } = await trx.rawQuery(
+      `SELECT v.idx, r.id
+       FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
+       JOIN releases r ON r.identity_key = v.key AND r.group_id <> ?
+       WHERE r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0)
+       ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
+      [JSON.stringify(open), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
+    )
+    if (candidates.length === 0) return
+    const ids: number[] = [...new Set<number>(candidates.map((c: { id: string }) => Number(c.id)))]
+    const names = new Map<number, Set<string>>()
+    const { rows: chunks } = await trx.rawQuery(`SELECT release_id, payload FROM release_chunks WHERE release_id = ANY(?)`, [ids])
+    for (const c of chunks) {
+      const set = names.get(Number(c.release_id)) ?? new Set<string>()
+      for (const f of decodeChunk(c.payload)) set.add(f.name)
+      names.set(Number(c.release_id), set)
+    }
+    for (const c of candidates as { idx: number; id: string }[]) {
+      if (joined.has(c.idx)) continue
+      const have = names.get(Number(c.id)) ?? new Set<string>()
+      const incoming = decodeChunk(part[c.idx].payload).map((f) => f.name)
+      if (incoming.every((name) => !have.has(name))) {
+        joined.set(c.idx, Number(c.id))
+        for (const name of incoming) have.add(name)
+        names.set(Number(c.id), have)
+      }
+    }
+  }
+
   /**
    * Release aggregates in the catalog, one write-once chunk per release, and
    * probe candidates (a batch's, or files the fragment merger completed).
-   * Releases are sorted by key, so concurrent writers lock them in the same order.
+   * A chunk joins the release of its key whose articles the range's group
+   * owns and whose posting window is within UPLOAD_GAP_MS of its own, else
+   * starts one: re-uploads are releases of their own (decision d). The
+   * group is locked, so concurrent writers don't both start one.
    */
   async writeReleases(
     trx: TransactionClientContract,
@@ -104,38 +168,35 @@ class BatchStore {
     releases: ReleaseChunk[],
     candidates: ProbeCandidate[]
   ) {
-    const out = { newReleases: 0, candidates: 0, ids: new Map<string, number>() }
+    const out = { newReleases: 0, candidates: 0 }
-    const ids = out.ids
+    const ids = new Map<string, number>()
-    // Releases, sorted by key so concurrent batches lock them in the same order.
+    const slot = (key: string, upload: number) => `${key}\n${upload}`
-    const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key))
+    const { group_id: groupId } = await trx.from('ranges').where('id', rangeId).select('group_id').firstOrFail()
+    // One writer per group at a time: a chunk only matches releases of its group, and a lock per key
+    // ran out of shared memory (a picture group's batch has 50k keys; LOG.md 2026-10-09 (12)).
+    await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('releases:' || ?::text, 0))`, [groupId])
+    const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
     for (let i = 0; i < sorted.length; i += 1000) {
-      const chunk = sorted.slice(i, i + 1000)
+      const part = sorted.slice(i, i + 1000)
-      const { rows } = await trx.rawQuery(
+      const windows = part.map((r, idx) => ({ idx, key: r.key, first: r.postedAt, last: r.postedLast ?? r.postedAt }))
-        `INSERT INTO releases (release_key, name, source, poster, groups, posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2)
+      const { rows: matches } = await trx.rawQuery(
-           SELECT v.key, v.name, v.source, v.poster, v.groups, to_timestamp(v.posted_at / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1, v.first_article, v.last_article, v.readable, v.has_par2
+        `SELECT DISTINCT ON (v.idx) v.idx, r.id
-           FROM jsonb_to_recordset(?::jsonb) AS v(key text, name text, source text, poster text, groups jsonb, posted_at bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
+         FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
-           ON CONFLICT (release_key) DO UPDATE SET
+         JOIN releases r ON r.identity_key = v.key AND r.group_id = ?
-             files = releases.files + excluded.files,
+         WHERE v.first IS NULL OR r.posted_at IS NULL
-             segments = releases.segments + excluded.segments,
+            OR (r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0))
-             bytes = releases.bytes + excluded.bytes,
+         ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
-             chunks = releases.chunks + 1,
+        [JSON.stringify(windows), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
-             posted_at = least(releases.posted_at, excluded.posted_at),
+      )
-             file_total = coalesce(releases.file_total, excluded.file_total),
+      const joined = new Map<number, number>(matches.map((m: { idx: number; id: string }) => [m.idx, Number(m.id)]))
-             first_article = least(releases.first_article, excluded.first_article),
+      await this.joinAcrossGroups(trx, part, windows, groupId, joined)
-             last_article = greatest(releases.last_article, excluded.last_article),
+      const row = (r: ReleaseChunk) => ({
-             readable = releases.readable OR excluded.readable,
-             has_par2 = releases.has_par2 OR excluded.has_par2,
-             last_seen_at = now()
-           RETURNING id, release_key, (xmax = 0) AS inserted`,
-        [
-          JSON.stringify(
-            chunk.map((r) => ({
-              key: r.key,
         name: r.name,
         source: r.source,
         poster: r.poster,
         groups: r.groups,
-              posted_at: r.postedAt,
+        first: r.postedAt,
+        last: r.postedLast ?? r.postedAt,
         file_total: r.fileTotal,
         files: r.files,
         segments: r.segments,
@@ -144,20 +205,79 @@ class BatchStore {
         last_article: r.lastArticle,
         readable: r.readable,
         has_par2: r.hasPar2,
-            }))
+      })
-          ),
+      const updates = part.flatMap((r, idx) => (joined.has(idx) ? [{ id: joined.get(idx)!, ...row(r) }] : []))
-        ]
+      if (updates.length > 0) {
+        await trx.rawQuery(
+          `WITH v AS (SELECT * FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)),
+                a AS (SELECT id, sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, count(*) AS chunks, min(first) AS first, max(last) AS last,
+                             max(file_total) AS file_total, min(first_article) AS first_article, max(last_article) AS last_article, bool_or(readable) AS readable, bool_or(has_par2) AS has_par2
+                      FROM v GROUP BY id)
+           UPDATE releases r SET
+             files = r.files + a.files,
+             segments = r.segments + a.segments,
+             bytes = r.bytes + a.bytes,
+             chunks = r.chunks + a.chunks,
+             posted_at = least(r.posted_at, to_timestamp(a.first / 1000.0)),
+             last_posted_at = greatest(r.last_posted_at, to_timestamp(a.last / 1000.0)),
+             file_total = coalesce(r.file_total, a.file_total),
+             first_article = least(r.first_article, a.first_article),
+             last_article = greatest(r.last_article, a.last_article),
+             readable = r.readable OR a.readable,
+             has_par2 = r.has_par2 OR a.has_par2,
+             -- As it stands (the counter's files all found); settling makes it final.
+             complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
+             last_seen_at = now()
+           FROM a WHERE r.id = a.id`,
+          [JSON.stringify(updates)]
         )
-      for (const row of rows) {
+        part.forEach((r, idx) => joined.has(idx) && ids.set(slot(r.key, r.upload), joined.get(idx)!))
-        ids.set(row.release_key, Number(row.id))
+      }
-        if (row.inserted) out.newReleases++
+      const fresh = part.filter((_, idx) => !joined.has(idx))
+      if (fresh.length > 0) {
+        // The first upload of a key keeps the key; later ones are key:2, key:3, …
+        const { rows: counts } = await trx.rawQuery(`SELECT identity_key, count(*) AS n FROM releases WHERE identity_key = ANY(?) GROUP BY 1`, [[...new Set(fresh.map((r) => r.key))]])
+        const taken = new Map<string, number>(counts.map((c: { identity_key: string; n: string }) => [c.identity_key, Number(c.n)]))
+        const numbered = (r: ReleaseChunk) => {
+          const n = (taken.get(r.key) ?? 0) + 1
+          taken.set(r.key, n)
+          return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
+        }
+        // Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
+        const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
+        if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters])
+        let inserts = fresh.map(numbered)
+        // Writers of other groups number the same key at the same time (a spam post crossposted to 67
+        // groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.
+        for (let attempt = 0; inserts.length > 0; attempt++) {
+          if (attempt === 20) throw new Error(`No free release key for ${inserts[0].key}`)
+          const { rows } = await trx.rawQuery(
+            `INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
+             SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
+                    v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
+             FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
+             ON CONFLICT (release_key) DO NOTHING
+             RETURNING id, release_key`,
+            [groupId, JSON.stringify(inserts)]
+          )
+          const byReleaseKey = new Map<string, number>(rows.map((x: { id: string; release_key: string }) => [x.release_key, Number(x.id)]))
+          const missed: typeof inserts = []
+          for (const r of inserts) {
+            const id = byReleaseKey.get(r.release_key)
+            if (id === undefined) missed.push(r)
+            else ids.set(slot(r.key, r.upload), id)
+          }
+          out.newReleases += rows.length
+          const chunks = new Map(fresh.map((r) => [slot(r.key, r.upload), r]))
+          inserts = missed.map((r) => numbered(chunks.get(slot(r.key, r.upload))!))
+        }
       }
     }
 
     // One write-once chunk per release.
     for (let i = 0; i < sorted.length; i += 500) {
       const rows = sorted.slice(i, i + 500).map((r) => ({
-        release_id: ids.get(r.key)!,
+        release_id: ids.get(slot(r.key, r.upload))!,
         range_id: rangeId,
         batch_from: batchFrom,
         files: r.files,
@@ -167,12 +287,12 @@ class BatchStore {
       if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
     }
 
-    const kept = candidates.filter((c) => ids.has(c.releaseKey))
+    const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
     for (let i = 0; i < kept.length; i += 2000) {
       const rows = kept
         .slice(i, i + 2000)
         .map((c) => ({
-          release_id: ids.get(c.releaseKey)!,
+          release_id: ids.get(slot(c.releaseKey, c.upload ?? 0))!,
           kind: c.kind,
           message_id: c.messageId,
           file: c.file,

app/services/catalog_space.ts

@@ -0,0 +1,38 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import coordinatorConfig from '#config/coordinator'
+
+/** How often the catalog's size is measured (pg_tablespace_size walks its files). */
+const CHECK_MS = 60_000
+
+/**
+ * The catalog's room on the NVMe (CATALOG_MAX_NVME_GB, like voyager 1's
+ * BACKFILL_MAX_NVME_GB): once its tablespace (the database's
+ * default_tablespace) is over the cap, only the tail and the stored ranges
+ * (priority below 0) are leased and prefetched, and history waits. After
+ * reset 24 the catalog grew ~3.7 GB an hour, root down to 31 GB free with
+ * 600k history ranges to go (LOG.md 2026-10-14 (53)). Off when unset.
+ */
+class CatalogSpace {
+  private checkedAt = 0
+  private over = false
+
+  async full(): Promise<boolean> {
+    const cap = coordinatorConfig.catalogMaxNvmeGb
+    if (!cap) return false
+    if (Date.now() - this.checkedAt < CHECK_MS) return this.over
+    this.checkedAt = Date.now()
+    const { rows } = await db.rawQuery(
+      `SELECT pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
+    )
+    const bytes = Number(rows[0].bytes)
+    const was = this.over
+    this.over = bytes > cap * 1e9
+    if (this.over !== was) {
+      logger.info(`Catalog ${(bytes / 1e9).toFixed(1)} GB of ${cap}: history ${this.over ? 'paused (tail and stored ranges only)' : 'resumed'}.`)
+    }
+    return this.over
+  }
+}
+
+export default new CatalogSpace()

app/services/corpus.ts

@@ -0,0 +1,90 @@
+import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { unzip } from 'node:zlib'
+import { promisify } from 'node:util'
+import coordinatorConfig from '#config/coordinator'
+import type { ChunkSpan } from '#services/spool'
+
+const inflate = promisify(unzip)
+
+export interface CorpusRange {
+  group: string
+  start: number
+  end: number
+}
+
+/**
+ * Stored XOVER chunks of the ranges marked `capture` (REBUILD.md, corpus):
+ * `<dir>/<group>/<start>-<end>/<nnnn>-<from>-<to>.gz` with a range.json once
+ * every chunk is there, the layout edge-bench reads (edge/fixtures). The
+ * grouping sweeps replay them with each build's rules instead of waiting for
+ * a re-index; the prefetcher reads a stored chunk instead of fetching it
+ * again. Off without CORPUS_DIR.
+ */
+class Corpus {
+  get dir(): string | undefined {
+    return coordinatorConfig.corpus.dir
+  }
+
+  private rangeDir(range: CorpusRange) {
+    return join(this.dir!, range.group, `${range.start}-${range.end}`)
+  }
+
+  private path(range: CorpusRange, span: ChunkSpan) {
+    const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
+    return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
+  }
+
+  /** Whether every chunk of the range is stored (its range.json). */
+  async complete(range: CorpusRange): Promise<boolean> {
+    if (!this.dir) return false
+    return access(join(this.rangeDir(range), 'range.json')).then(
+      () => true,
+      () => false
+    )
+  }
+
+  /** Whether one chunk is stored. */
+  async has(range: CorpusRange, span: ChunkSpan): Promise<boolean> {
+    if (!this.dir) return false
+    return access(this.path(range, span)).then(
+      () => true,
+      () => false
+    )
+  }
+
+  /** A stored chunk, or null. */
+  async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
+    if (!this.dir) return null
+    return readFile(this.path(range, span)).catch(() => null)
+  }
+
+  /**
+   * Stores a chunk that inflates (a reply cut short isn't kept), and the
+   * range.json once the range is complete (`total` chunks).
+   */
+  async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
+    if (!this.dir) return
+    try {
+      await inflate(data)
+    } catch {
+      return
+    }
+    const dir = this.rangeDir(range)
+    const file = this.path(range, span)
+    await mkdir(dir, { recursive: true })
+    await writeFile(`${file}.part`, data)
+    await rename(`${file}.part`, file)
+    const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
+    if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
+  }
+
+  /** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
+  async removeChunk(range: CorpusRange, span: ChunkSpan) {
+    if (!this.dir) return
+    await rm(this.path(range, span), { force: true })
+    await rm(join(this.rangeDir(range), 'range.json'), { force: true })
+  }
+}
+
+export default new Corpus()

app/services/fragment_merger.ts

@@ -3,7 +3,7 @@ import BatchStore from '#services/batch_store'
 import ChunkPartitions from '#services/chunk_partitions'
 import RuleSets from '#services/rule_sets'
 import { RuleSet } from '#lib/rules/index'
-import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
+import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, uploadsOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
 import type { AggregatedFile } from '#lib/files/index'
 import type { EdgeSegment } from '#lib/edge/index'
 
@@ -83,12 +83,14 @@ class FragmentMerger {
     }
     const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
     for (const [key, { files, rangeId }] of byRelease) {
-      const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId })))
+      const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
       if (verdict.rule !== null) continue
-      releases.push({ rangeId, chunk: chunkOf(key, files), candidates: candidatesOf(key, files) })
+      uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
     }
     await ChunkPartitions.ensure()
     await db.transaction(async (trx) => {
+      // In key order (batches lock their group; the merger only ever waits on one batch at a time).
+      releases.sort((a, b) => a.chunk.key.localeCompare(b.chunk.key) || a.chunk.upload - b.chunk.upload)
       for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
       const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
       for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()

app/services/leases.ts

@@ -2,6 +2,8 @@ import db from '@adonisjs/lucid/services/db'
 import coordinatorConfig from '#config/coordinator'
 import RuleSets from '#services/rule_sets'
 import Spool from '#services/spool'
+import Corpus from '#services/corpus'
+import CatalogSpace from '#services/catalog_space'
 import type { Lease } from '#lib/edge/index'
 
 export class LeaseLost extends Error {}
@@ -15,19 +17,31 @@ export class LeaseLost extends Error {}
  * prefetched (LOG.md 2026-10-09). A lease whose worker went quiet for
  * LEASE_TTL_SECONDS is taken over and resumes at its resume point.
  */
+/**
+ * A group's batches wait for each other (the batch store locks the group): more workers than this on one group only
+ * queue. Eight on one picture group's gap ranges (priority -1, in id order) held the node API's batches for minutes
+ * (LOG.md 2026-10-16 (68)).
+ */
+const MAX_LEASES_PER_GROUP = 2
+
 class Leases {
   async lease(nodeId: number): Promise<Lease | null> {
     const rules = await RuleSets.current()
+    // The catalog at its cap on the NVMe: the tail and the stored ranges only (CatalogSpace).
+    const history = (await CatalogSpace.full()) ? 'AND priority < 0' : ''
     const take = (spooled: boolean) =>
       db.rawQuery(
         `UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
          WHERE r.id = (
            SELECT id FROM ranges
-           WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id
+           WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id ${history}
              AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
+             AND (SELECT count(*) FROM ranges l
+                  WHERE l.status = 'leased' AND l.group_id = ranges.group_id AND l.id <> ranges.id
+                    AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
            ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
          RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
-        [nodeId, rules.version, coordinatorConfig.leaseTtlSeconds]
+        [nodeId, rules.version, coordinatorConfig.leaseTtlSeconds, coordinatorConfig.leaseTtlSeconds]
       )
     let { rows } = await take(true)
     if (rows.length === 0) ({ rows } = await take(false))
@@ -66,8 +80,23 @@ class Leases {
   async corrupt(rangeId: number, nodeId: number, from: number) {
     const range = await this.leased(rangeId, nodeId)
     const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find((s) => s.from === from)
-    if (span) await Spool.removeChunk(rangeId, span)
+    if (span) {
-    await db.from('ranges').where('id', rangeId).update({ status: 'planned', node_id: null })
+      await Spool.removeChunk(rangeId, span)
+      if (range.capture) {
+        const group = await db.from('groups').where('id', range.group_id).select('name').first()
+        await Corpus.removeChunk({ group: group.name, start: Number(range.start_id), end: Number(range.end_id) }, span)
+      }
+    }
+    const plain: number[] = (range.plain_chunks ?? []).map(Number)
+    // Reported once already: the compressed copy may have been in flight (or the prefetcher's targets not
+    // refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.
+    if (plain.filter((p) => p === from).length >= 2) {
+      // Fetched plain (deflated here) and still not inflating: not the server's compression; set aside.
+      await db.from('ranges').where('id', rangeId).update({ status: 'failed', node_id: null, error: `chunk ${from} doesn't inflate, also fetched plain` })
+      return
+    }
+    // The server's compressed reply is cut short (the same bytes every time): fetched plain next time.
+    await db.rawQuery(`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`, [from, rangeId])
   }
 
   /** A stopping worker gives its range back; the next one resumes it. */

app/services/nntp_pool.ts

@@ -6,20 +6,43 @@ import { XoverClient } from '#lib/nntp/index'
 /**
  * The coordinator's header connections (REBUILD.md, decision e: workers
  * have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
- * that failed is closed and replaced on the next use.
+ * that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
+ * replaced on the next use.
  */
 class NntpPool {
   private idle: XoverClient[] = []
   private busy = new Set<XoverClient>()
   private open = 0
   private waiting: ((client: XoverClient) => void)[] = []
+  private openedAt = new Map<XoverClient, number>()
+  /** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
+  private retireAt = new Map<XoverClient, number>()
 
   get size() {
     return coordinatorConfig.nntpConnections
   }
 
   async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
-    const client = await this.acquire()
+    return this.run(await this.acquire(), work)
+  }
+
+  /** As use(), if a connection is free or can be opened now; else null, without waiting. */
+  tryUse<T>(work: (client: XoverClient) => Promise<T>): Promise<T> | null {
+    if (this.idle.length === 0 && this.open >= this.size) return null
+    return this.acquire().then((client) => this.run(client, work))
+  }
+
+  /** Connections with a command in flight (the prefetcher's, the planner's). */
+  get busyCount() {
+    return this.busy.size
+  }
+
+  /** How long ago the connection was opened, in ms. */
+  ageOf(client: XoverClient): number {
+    return Date.now() - (this.openedAt.get(client) ?? Date.now())
+  }
+
+  private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
     this.busy.add(client)
     let broken = false
     try {
@@ -39,7 +62,7 @@ class NntpPool {
     if (this.open < this.size) {
       this.open++
       try {
-        return await XoverClient.connect({
+        const client = await XoverClient.connect({
           host: env.get('NNTP_HOST')!,
           port: env.get('NNTP_PORT', 563),
           user: env.get('NNTP_USER'),
@@ -48,6 +71,10 @@ class NntpPool {
           compress: true,
           timeoutMs: coordinatorConfig.nntpTimeoutMs,
         })
+        this.openedAt.set(client, Date.now())
+        const maxMs = coordinatorConfig.nntpConnectionMaxMs
+        if (maxMs > 0) this.retireAt.set(client, Date.now() + maxMs * (0.75 + Math.random() / 2))
+        return client
       } catch (error) {
         this.open--
         throw error
@@ -57,8 +84,10 @@ class NntpPool {
   }
 
   private release(client: XoverClient, broken: boolean) {
-    if (broken) {
+    if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
       this.open--
+      this.openedAt.delete(client)
+      this.retireAt.delete(client)
       try {
         client.close()
       } catch (error: any) {
@@ -75,9 +104,45 @@ class NntpPool {
   }
 
   /** Closes every connection, also those in use: their commands fail. */
+  private plain: Promise<XoverClient> | null = null
+  private plainQueue: Promise<unknown> = Promise.resolve()
+
+  /**
+   * One connection without XFEATURE COMPRESS GZIP, opened when first needed,
+   * one command at a time: for chunks whose compressed reply the server
+   * cuts short (Leases.corrupt). Its replies are deflated here.
+   */
+  async usePlain<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
+    const run = this.plainQueue.then(async () => {
+      this.plain ??= XoverClient.connect({
+        host: env.get('NNTP_HOST')!,
+        port: env.get('NNTP_PORT', 563),
+        user: env.get('NNTP_USER'),
+        password: env.get('NNTP_PASSWORD'),
+        secure: env.get('NNTP_SECURE', true),
+        compress: false,
+        timeoutMs: coordinatorConfig.nntpTimeoutMs,
+      })
+      const client = await this.plain
+      try {
+        return await work(client)
+      } catch (error) {
+        this.plain = null
+        client.close()
+        throw error
+      }
+    })
+    this.plainQueue = run.catch(() => {})
+    return run
+  }
+
   async close() {
+    void this.plain?.then((c) => c.close()).catch(() => {})
+    this.plain = null
     for (const client of [...this.idle.splice(0), ...this.busy]) client.close()
     this.busy.clear()
+    this.openedAt.clear()
+    this.retireAt.clear()
     this.open = 0
   }
 }

app/services/nzb_service.ts

@@ -1,25 +1,34 @@
+import { gunzipSync } from 'node:zlib'
 import db from '@adonisjs/lucid/services/db'
-import { decodeChunk } from '#lib/chunks/index'
+import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
 import { buildNzb } from '#lib/nzb/index'
 
 /**
- * A release's NZB (REBUILD.md, part 9), from its chunks in the chunk store:
+ * A release's files and NZB (REBUILD.md, part 9), from its chunks in the
- * every chunk decoded, its files in name order. A file stored twice (the same
+ * chunk store: every chunk decoded, its files in name order. A file stored
- * name and first article, e.g. a range read again) is taken once.
+ * twice (the same name and first article, e.g. a range read again) is taken
+ * once.
  */
 class NzbService {
-  async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
+  async files(releaseId: number): Promise<ChunkFile[]> {
-    const release = await db.from('releases').where('id', releaseId).select('name').first()
-    if (!release) return null
     const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
-    const files = new Map<string, ReturnType<typeof decodeChunk>[number]>()
+    const files = new Map<string, ChunkFile>()
     for (const chunk of chunks) {
       for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
     }
+    return [...files.values()].sort((a, b) => a.name.localeCompare(b.name))
+  }
+
+  async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
+    const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
+    if (!release) return null
+    // A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
+    if (release.source === 'spot') {
+      const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
+      if (spot) return { name: release.name, xml: gunzipSync(spot.nzb).toString('utf8') }
+    }
     const xml = buildNzb(
-      [...files.values()]
+      (await this.files(releaseId)).map((f) => ({
-        .sort((a, b) => a.name.localeCompare(b.name))
-        .map((f) => ({
         name: f.name,
         poster: f.poster,
         date: f.postedAt ?? 0,

app/services/par2_merger.ts

@@ -0,0 +1,315 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { par2ReleaseName } from '#lib/release/index'
+import { looksRandom } from '#lib/release/random'
+import { SidecarFolder } from '#services/sidecar_folder'
+import { Par2Pruner } from '#services/par2_pruner'
+
+/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
+const MAX_SPREAD_MS = 48 * 3600 * 1000
+/**
+ * A set held by more releases nearby than this is one posted again and again, not one upload's par2 (every
+ * "kantodoNNN.bin" test upload carries the same set: 64 releases became one of 631 files; LOG.md 2026-10-12 (30)).
+ */
+const MAX_HOLDERS = 4
+/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
+const MAX_FILES_PER_SET_FILE = 3
+const SLACK_FILES = 10
+/** Between rounds. */
+const ROUND_MS = 60_000
+/** A round looks back this far past the previous one's start (rows committed while it ran). */
+const OVERLAP_MS = 60_000
+/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
+const FOLD_EVERY_MS = 5 * 60_000
+/** Between par2_files prunes (complete and judged sets' file lists). */
+const PRUNE_EVERY_MS = 60 * 60_000
+/** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
+const CURSOR = 'merger'
+
+export interface MergeRound {
+  sets: number
+  merges: number
+  releases: number
+  /** The inner-name pass: stems checked, merges, releases in them. */
+  stems: number
+  stemMerges: number
+  stemReleases: number
+  /** The sidecar fold: candidates looked at, sidecar-only ones among them, folded into their upload. */
+  folded?: { looked: number; sidecars: number; folded: number }
+}
+
+/**
+ * A lone inspection's inner stem: its yEnc name up to the first dot, when 16–64 letters and digits
+ * ("bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001"). The same expression as release_inspections_inner_stem_idx.
+ */
+const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
+
+interface Member {
+  id: number
+  release_key: string
+  group_id: number
+  files: number
+  posted_at: Date | null
+  readable: boolean
+  poster?: string | null
+  file_total?: number | null
+}
+
+/**
+ * The par2 merge (REBUILD.md, phase 2): releases whose files a par2 set
+ * describes are one upload. Obfuscated uploads post each file under a name of
+ * its own ("[9/20] - "14f5999a7bd4680b0720e524cac0b3b5"", hep: 13,232
+ * one-file releases), so the header pass can't group them; the prober reads
+ * each such file's first article (a `lone` candidate) and stores the MD5 of
+ * its first 16 KiB, and a par2 body's file list holds the same hash per file,
+ * whatever the file was renamed to. Releases in one group, posted within
+ * MAX_SPREAD_MS of a release that holds the set's par2, become the one with
+ * the most files: its chunks, probe candidates, inspections and hashes, the
+ * sums of its counts, and when none of them has a readable name the name the
+ * par2's file names share. Each merge is kept in release_merges.
+ *
+ * A second pass merges by inner name: the yEnc name in a lone file's first
+ * article keeps the upload's random stem when the subject names each file
+ * apart ("[1/7] - "471044d8…"" holds "bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001";
+ * hep, hephaestus, gyges: every stem under one counter total; LOG.md
+ * 2026-10-12 (32)). Releases whose lone inspections share such a stem, in one
+ * group, by one poster, under the same counter total and within MAX_SPREAD_MS,
+ * are one upload. Only random stems: a readable one is grouped from the
+ * headers already, and a repost would share it.
+ */
+export class Par2Merger {
+  private stopped = false
+  private folder = new SidecarFolder((trx, found, key, via) => this.merge(trx, found, key, via))
+  /** Null until read from worker_cursors (1970 the first time: everything). */
+  private since: Date | null = null
+
+  stop() {
+    this.stopped = true
+  }
+
+  async run() {
+    let prunedAt = 0
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        // First, at start too: a prune needn't wait for a long round.
+        if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
+          prunedAt = Date.now()
+          const pruned = await new Par2Pruner().round()
+          if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
+        }
+        const round = await this.round()
+        if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
+        if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
+        if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
+      } catch (error: any) {
+        logger.error(`Par2 merge round failed: ${error.message}`)
+      }
+      for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  /** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
+  async round(): Promise<MergeRound> {
+    const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
+    if (!this.since) {
+      const saved = await db.from('worker_cursors').where('name', CURSOR).first()
+      this.since = saved ? new Date(saved.at) : new Date(0)
+    }
+    const since = new Date(this.since.getTime() - OVERLAP_MS)
+    // First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
+    // par2 pass (hours when that starts over).
+    const folded = await this.folder.round()
+    let foldedAt = Date.now()
+    const foldAgain = async () => {
+      if (Date.now() - foldedAt < FOLD_EVERY_MS) return
+      const more = await this.folder.round()
+      folded.looked += more.looked
+      folded.sidecars += more.sidecars
+      folded.folded += more.folded
+      foldedAt = Date.now()
+      if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
+    }
+    const { rows } = await db.rawQuery(
+      `SELECT set_id FROM par2_sets WHERE created_at >= ?
+       UNION
+       SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
+      [since, since]
+    )
+    const round: MergeRound = { sets: rows.length, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
+    for (const { set_id: setId } of rows) {
+      if (this.stopped) return round
+      await foldAgain()
+      // One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
+      const merged = await this.mergeSet(setId.trim()).catch((error) => {
+        logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
+        return 0
+      })
+      if (merged > 0) {
+        round.merges++
+        round.releases += merged + 1
+      }
+    }
+    const { rows: stems } = await db.rawQuery(
+      `SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i WHERE i.kind = 'lone' AND i.created_at >= ? AND ${INNER_STEM} IS NOT NULL`,
+      [since]
+    )
+    for (const { stem } of stems) {
+      if (this.stopped) return round
+      if (!looksRandom(stem)) continue
+      round.stems++
+      const merged = await this.mergeStem(stem).catch((error) => {
+        logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
+        return 0
+      })
+      if (merged > 0) {
+        round.stemMerges++
+        round.stemReleases += merged + 1
+      }
+    }
+    // Only after a whole round: a restart halfway goes on from the round before.
+    this.since = now[0].at
+    await db.rawQuery(
+      `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
+      [CURSOR, this.since]
+    )
+    return round
+  }
+
+  /** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
+  async mergeStem(stem: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows } = await trx.rawQuery(
+        `SELECT DISTINCT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster, r.file_total
+         FROM release_inspections i JOIN releases r ON r.id = i.release_id
+         WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
+        [stem]
+      )
+      const sorted: Member[] = rows.sort((a: Member, b: Member) => (a.posted_at ? new Date(a.posted_at).getTime() : 0) - (b.posted_at ? new Date(b.posted_at).getTime() : 0))
+      const anchor = sorted[0]
+      if (!anchor) return 0
+      const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
+      const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
+      const near = (r: Member) =>
+        r.group_id === anchor.group_id &&
+        (r.poster ?? '') === (anchor.poster ?? '') &&
+        (total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
+        (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
+      const found = new Map<string, Member>()
+      for (const r of sorted.filter(near)) found.set(String(r.id), r)
+      if (found.size < 2) return 0
+      // No more files than the counter says, with room for par2 volumes posted under it.
+      const counted = Math.max(0, ...[...found.values()].map((r) => total(r) ?? 0))
+      const files = [...found.values()].reduce((n, m) => n + Number(m.files), 0)
+      if (counted > 0 && files > counted + SLACK_FILES) return 0
+      return this.merge(trx, found, stem, 'inner')
+    })
+  }
+
+  /** Merges the releases of one set; how many went into another. */
+  async mergeSet(setId: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows: holders } = await trx.rawQuery(
+        `SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
+         WHERE s.set_id = ? ORDER BY r.id`,
+        [setId]
+      )
+      const anchor: Member | undefined = holders[0]
+      if (!anchor) return 0
+      const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
+      const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
+      const { rows: matches } = await trx.rawQuery(
+        `SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable
+         FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length JOIN releases r ON r.id = h.release_id
+         WHERE f.set_id = ?`,
+        [setId]
+      )
+      // A file of the set that matches more than one release nearby is ambiguous: files that share their first
+      // 16 KiB and size (131 uploads of 1 GiB test files "kantodoNNN.bin" matched one set's single file and became
+      // one release of 1,284 files; LOG.md 2026-10-12 (16)). A plain repost of the same file stays apart too.
+      const byFile = new Map<string, Map<string, Member>>()
+      for (const m of matches.filter(near)) {
+        const releases = byFile.get(m.file_id) ?? new Map<string, Member>()
+        releases.set(String(m.id), m)
+        byFile.set(m.file_id, releases)
+      }
+      const nearHolders = holders.filter(near)
+      if (nearHolders.length > MAX_HOLDERS) return 0
+      const found = new Map<string, Member>()
+      for (const r of nearHolders) found.set(String(r.id), r)
+      for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r)
+      if (found.size < 2) return 0
+      const { rows: described } = await trx.rawQuery(`SELECT count(*) AS n FROM par2_files WHERE set_id = ?`, [setId])
+      const setFiles = Math.max(Number(described[0].n), 1)
+      if ([...found.values()].reduce((n, m) => n + Number(m.files), 0) > MAX_FILES_PER_SET_FILE * setFiles + SLACK_FILES) return 0
+      return this.merge(trx, found, setId, 'par2')
+    })
+  }
+
+  /**
+   * Merges the found releases into the one with the most files (named by the par2 set's files when none
+   * is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
+   */
+  private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar'): Promise<number> {
+    {
+      // Locked in id order (the batch store and other rounds take the same rows).
+      const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
+      const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!)
+      if (members.length < 2) return 0
+      const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a))
+      const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id))
+      const name = via === 'par2' && members.every((m) => !m.readable) ? await this.setName(trx, key) : undefined
+
+      await trx.rawQuery(
+        `INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
+         SELECT id, release_key, ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
+        [target.id, key, via, others]
+      )
+      for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
+        await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      }
+      // A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
+      // round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
+      await trx.rawQuery(
+        `DELETE FROM par2_sets p WHERE p.release_id = ANY(?)
+           AND (EXISTS (SELECT 1 FROM par2_sets t WHERE t.set_id = p.set_id AND t.release_id = ?)
+             OR EXISTS (SELECT 1 FROM par2_sets o WHERE o.set_id = p.set_id AND o.release_id = ANY(?) AND o.release_id < p.release_id))`,
+        [others, target.id, others]
+      )
+      await trx.rawQuery(`UPDATE par2_sets SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      await trx.rawQuery(
+        `UPDATE releases t SET
+           files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
+           first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
+           posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
+           first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
+           readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
+           file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
+           name = coalesce(nullif(?, ''), t.name)
+         FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
+                 min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
+                 max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
+                 bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
+                 max(file_total) AS file_total, max(inner_name) AS inner_name
+               FROM releases WHERE id = ANY(?)) s
+         WHERE t.id = ?`,
+        [name ?? '', others, target.id]
+      )
+      // Completeness as the settler decides it, once settled.
+      await trx.rawQuery(
+        `UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
+        [target.id]
+      )
+      await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
+      return others.length
+    }
+  }
+
+  /** The name a set's files share (par2ReleaseName), for an upload none of whose names is readable. */
+  private async setName(trx: any, setId: string): Promise<string | undefined> {
+    const { rows } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
+    const name = par2ReleaseName(rows.map((r: { name: string }) => r.name))
+    // A par2 made after the renaming names the random names again.
+    return name && !looksRandom(name) ? name.slice(0, 255) : undefined
+  }
+}

app/services/par2_pruner.ts

@@ -0,0 +1,81 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
+const SETTLED_MS = 2 * 86400_000
+/** Sets per statement. */
+const BATCH = 20_000
+/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
+const CURSOR = 'par2-prune'
+/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
+const JUNK = ['dead', 'fake', 'repeated']
+
+export interface PruneRound {
+  complete: number
+  judged: number
+  rows: number
+}
+
+/**
+ * Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
+ * 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
+ * done when
+ * - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
+ *   set (all of it merged into one);
+ * - judged: every release holding it has a junk verdict (dead, fake, repeated).
+ * par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
+ * par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
+ */
+export class Par2Pruner {
+  async round(): Promise<PruneRound> {
+    const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
+    const saved = await db.from('worker_cursors').where('name', CURSOR).first()
+    let since: Date = saved ? new Date(saved.at) : new Date(0)
+    const until = new Date(Date.now() - SETTLED_MS)
+    for (;;) {
+      const { rows: sets } = await db.rawQuery(
+        `SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
+         HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
+        [since, until, BATCH]
+      )
+      if (sets.length === 0) break
+      const ids = sets.map((s: { set_id: string }) => s.set_id)
+      const { rows: done } = await db.rawQuery(
+        `SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
+         WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
+           AND NOT EXISTS (
+             SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
+               SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
+               WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
+        [ids]
+      )
+      round.complete += done.length
+      round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
+      since = sets.at(-1).at
+      await db.rawQuery(
+        `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
+        [CURSOR, since]
+      )
+      if (sets.length < BATCH) break
+    }
+    // Judged: no cursor (verdicts come and go); only sets with rows left.
+    const { rows: judged } = await db.rawQuery(
+      `SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
+       WHERE v.rule = ANY(?)
+         AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
+         AND NOT EXISTS (
+           SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
+           WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
+      [JUNK, JUNK]
+    )
+    round.judged = judged.length
+    const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
+    for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
+    return round
+  }
+
+  private async drop(setIds: string[]): Promise<number> {
+    if (setIds.length === 0) return 0
+    const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
+    return result.rowCount ?? 0
+  }
+}

app/services/planner.ts

@@ -54,8 +54,9 @@ class Planner {
     return rows.length
   }
 
+  /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */
   async planAll(): Promise<{ groups: number; ranges: number }> {
-    const groups = await db.from('groups').where('active', true).orderBy('priority').select('id', 'name')
+    const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name')
     let ranges = 0
     for (const group of groups) {
       // One group failing (e.g. gone from the server) doesn't stop the others.
@@ -67,6 +68,78 @@ class Planner {
     }
     return { groups: groups.length, ranges }
   }
+
+  /**
+   * The live tail: ranges for the articles after each planned group's last
+   * range, at the tail's priority (before any history). The newest article
+   * numbers come from LIST ACTIVE, one command per name prefix (GROUP per
+   * group took minutes for 1000 groups).
+   */
+  async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
+    const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
+    if (groups.length === 0) return { groups: 0, ranges: 0 }
+    const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
+    const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
+    const highs = new Map<string, number>()
+    await NntpPool.use(async (client) => {
+      for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
+        for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
+      }
+    })
+    const rows = tailRanges(
+      groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
+      now,
+      { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
+    )
+    if (rows.length === 0) return { groups: 0, ranges: 0 }
+    await db.transaction(async (trx) => {
+      await trx.table('ranges').multiInsert(rows)
+      for (const id of new Set(rows.map((r) => r.group_id))) {
+        const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
+        await trx.from('groups').where('id', id).update({ high, planned_at: now })
+      }
+    })
+    return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
+  }
+}
+
+/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
+export function wildmatOf(name: string): string {
+  const parts = name.split('.')
+  return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name
+}
+
+export interface TailGroup {
+  id: number
+  /** End of its last range (null: none, then it has no tail yet). */
+  plannedTo: number | null
+  /** When its last range was planned. */
+  plannedAt: Date
+  /** The server's newest article (null: not listed). */
+  high: number | null
+}
+
+/**
+ * The tail's new ranges: per group, the articles after its last range, in
+ * ranges of at most `rangeArticles`, once there are `minArticles` of them or
+ * the group's last planning is `maxDelayMinutes` ago.
+ */
+export function tailRanges(
+  groups: TailGroup[],
+  now: Date,
+  options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
+) {
+  const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
+  for (const g of groups) {
+    if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
+    const waiting = g.high - g.plannedTo
+    const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
+    if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
+    for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
+      rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
+    }
+  }
+  return rows
 }
 
 export default new Planner()

app/services/prefetcher.ts

@@ -3,16 +3,34 @@ import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import NntpPool from '#services/nntp_pool'
 import Spool, { type ChunkSpan } from '#services/spool'
+import Corpus from '#services/corpus'
+import CatalogSpace from '#services/catalog_space'
+import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
 
 interface Target {
   id: number
   group: string
+  /** Chunk starts to fetch without compression (ranges.plain_chunks). */
+  plain: Set<number>
   /** Chunks not spooled yet, in order. */
   missing: ChunkSpan[]
   /** All of the range's chunks, from its start. */
   total: number
+  start: number
+  end: number
+  /** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
+  capture: boolean
+  /** Fetched into the corpus only, not the spool: a capture range no worker leases soon (corpus.ahead). */
+  corpusOnly?: boolean
 }
 
+/** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
+const STALL_SAMPLE_MS = 15_000
+const STALL_SAMPLES = 60_000 / STALL_SAMPLE_MS
+
+/** Connection ages the report counts slow first requests by, in minutes. */
+const AGE_BUCKETS = [2, 10, Infinity]
+
 /** How often the targets are read again from the database. */
 const REFRESH_MS = 2000
 /** How often the spool's size is measured (a stat per file). */
@@ -21,11 +39,13 @@ const SIZE_MS = 30_000
 /**
  * Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
  * Every connection takes the next missing chunk: the leased ranges' first
- * (a worker waits for them; all connections work on the same range, in
+ * (a worker waits for them), then the next planned ranges, with at most
- * order), then the next planned ranges, about SPOOL_AHEAD_PER_LEASE per
+ * PREFETCH_PER_RANGE connections on a range and PREFETCH_PER_GROUP on a
+ * group (Eweka answers faster spread out), about SPOOL_AHEAD_PER_LEASE per
  * active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
  * connection fills ~26 k overview lines/s, so a range on one connection
- * kept its worker waiting (LOG.md 2026-10-09).
+ * kept its worker waiting (LOG.md 2026-10-09). A chunk that takes long is
+ * asked again on a spare connection (fetchHedged).
  */
 export class Prefetcher {
   /**
@@ -37,22 +57,75 @@ export class Prefetcher {
 
   private targets: Target[] = []
   private claimed = new Set<string>()
+  /** Chunks being fetched per range and per group (spool.perRange, spool.perGroup). */
+  private perRange = new Map<number, number>()
+  private perGroup = new Map<string, number>()
   private running = 0
   private refreshedAt = 0
   private stopped = false
   private spoolBytes = 0
   private sizedAt = 0
   /** Fetched since the last report: chunks, articles, ms spent fetching. */
-  private stats = { chunks: 0, articles: 0, fetchMs: 0, since: Date.now() }
+  private stats = this.freshStats()
+  /** Loops in a fetch or a spool write right now. */
+  private fetchingNow = 0
+  private writingNow = 0
+  private sampler?: NodeJS.Timeout
+  /** Requests in flight: what, since when, on which connection (the report names the oldest). */
+  private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>()
+  /** Capture ranges found complete in the corpus (not looked at again). */
+  private stored = new Set<number>()
+
+  private freshStats() {
+    return {
+      chunks: 0,
+      /** Chunks read from the corpus instead of fetched. */
+      stored: 0,
+      /** Chunks fetched into the corpus only (corpus.ahead). */
+      captured: 0,
+      articles: 0,
+      fetchMs: 0,
+      hedged: 0,
+      hedgeWon: 0,
+      /** First requests per connection-age bucket, and how many of them were slow. */
+      byAge: AGE_BUCKETS.map(() => ({ requests: 0, slow: 0 })),
+      /** Spool writes and the range's update, ms. */
+      writeMs: 0,
+      /** Per answered request, ms: GROUP, XOVER until its status line, the transfer; transfers with a silence over 5 s. */
+      requests: 0,
+      groupMs: 0,
+      replyMs: 0,
+      bodyMs: 0,
+      silent: 0,
+      /** Once a second: loops fetching, writing, not running; pool connections with a command in flight. */
+      samples: 0,
+      fetching: 0,
+      writing: 0,
+      idle: 0,
+      inFlight: 0,
+      inFlightMin: Infinity,
+      /** Requests given up as stalled (nntpStallBytesPerMinute). */
+      deadlines: 0,
+      ranges: 0,
+      groups: 0,
+      since: Date.now(),
+    }
+  }
+
+  /** The prefetcher's loops: the pool less the connections kept for hedging. */
+  private get loops() {
+    return Math.max(1, NntpPool.size - coordinatorConfig.nntpHedgeConnections)
+  }
 
   /** One round: refreshes the targets now and then, and puts idle connections to work. */
   async tick() {
+    this.sampler ??= setInterval(() => this.sample(), 1000).unref()
     if (Date.now() - this.refreshedAt >= REFRESH_MS) {
       this.refreshedAt = Date.now()
       this.targets = await this.readTargets()
     }
     this.report()
-    while (!this.stopped && this.running < NntpPool.size && this.peek()) {
+    while (!this.stopped && this.running < this.loops && this.peek()) {
       this.running++
       this.work()
         .catch((error) => logger.warn(`Prefetching: ${error.message}`))
@@ -62,6 +135,7 @@ export class Prefetcher {
 
   stop() {
     this.stopped = true
+    clearInterval(this.sampler)
   }
 
   get active() {
@@ -76,13 +150,35 @@ export class Prefetcher {
       const { target, span } = next
       try {
         const fetchedAt = Date.now()
-        const data = await NntpPool.use(async (client) => {
+        this.fetchingNow++
+        const fetch = async (client: Parameters<Parameters<typeof NntpPool.use>[0]>[0]) => {
           await client.group(target.group)
           return client.xoverRaw(span.from, span.to)
-        })
+        }
+        const corpusRange = { group: target.group, start: target.start, end: target.end }
+        // A range in the corpus is read from disk (a reset re-indexes it without the news server).
+        let data = target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
+        const stored = data !== null
+        try {
+          if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
+        } finally {
+          this.fetchingNow--
+        }
+        if (stored) {
+          this.stats.stored++
+        } else {
           this.stats.chunks++
           this.stats.articles += span.to - span.from + 1
           this.stats.fetchMs += Date.now() - fetchedAt
+          if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
+        }
+        if (target.corpusOnly) {
+          this.stats.captured++
+          continue
+        }
+        const writtenAt = Date.now()
+        this.writingNow++
+        try {
           const existed = await Spool.has(target.id, span)
           await Spool.write(target.id, span, data)
           if (existed) continue
@@ -92,37 +188,180 @@ export class Prefetcher {
             [target.id]
           )
           if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
+        } finally {
+          this.writingNow--
+          this.stats.writeMs += Date.now() - writtenAt
+        }
       } finally {
         this.claimed.delete(`${target.id}:${span.from}`)
+        this.count(target, -1)
       }
     }
   }
 
+  /**
+   * A chunk over a pooled connection. Not in after nntpHedgeAfterMs, it's
+   * asked again on a free connection (if there is one): Eweka stalls single
+   * requests, and the same chunk on another connection mostly comes in
+   * seconds (LOG.md 2026-10-09 (15)). The first answer wins; the other
+   * request can't be cancelled, so its connection is closed (the pool opens
+   * a fresh one).
+   */
+  private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
+    let age: number | undefined
+    const request = (start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null, first: boolean): Request<Buffer> | null => {
+      let client: XoverClient | undefined
+      let cancelled = false
+      const done = start(async (c) => {
+        // Given up before it got a connection: the connection goes back unused.
+        if (cancelled) return Buffer.alloc(0)
+        client = c
+        if (first) age = NntpPool.ageOf(c)
+        const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c }
+        this.inFlight.add(entry)
+        // Bytes in, sampled every STALL_SAMPLE_MS: less than nntpStallBytesPerMinute over the last minute is a stall.
+        let late = false
+        const samples: number[] = []
+        const watch = setInterval(() => {
+          samples.push(c.buffered)
+          if (samples.length <= STALL_SAMPLES) return
+          const minuteAgo = samples.shift()!
+          if (c.buffered - minuteAgo < coordinatorConfig.nntpStallBytesPerMinute) {
+            late = true
+            this.stats.deadlines++
+            c.close()
+          }
+        }, STALL_SAMPLE_MS)
+        let data: Buffer
+        let groupMs = 0
+        try {
+          const groupAt = performance.now()
+          await c.group(group)
+          groupMs = performance.now() - groupAt
+          data = await c.xoverRaw(span.from, span.to)
+        } catch (error) {
+          if (!late) throw error
+          throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`)
+        } finally {
+          clearInterval(watch)
+          this.inFlight.delete(entry)
+        }
+        if (!cancelled) {
+          const t = c.lastTiming
+          this.stats.requests++
+          this.stats.groupMs += groupMs
+          this.stats.replyMs += t.replyMs
+          this.stats.bodyMs += t.bodyMs
+          if (t.longestGapMs > 5000) this.stats.silent++
+        }
+        return data
+      })
+      if (!done) return null
+      done.catch(() => {})
+      return {
+        done,
+        // The request can't be called off on the wire: its connection is closed, and the pool opens a fresh one.
+        cancel: () => {
+          cancelled = true
+          client?.close()
+        },
+      }
+    }
+    const result = await hedge(
+      () => request((work) => NntpPool.use(work), true)!,
+      () => request((work) => NntpPool.tryUse(work), false),
+      coordinatorConfig.nntpHedgeAfterMs
+    )
+    if (age !== undefined) {
+      const bucket = this.stats.byAge[AGE_BUCKETS.findIndex((minutes) => age! < minutes * 60_000)]
+      bucket.requests++
+      if (result.late) bucket.slow++
+    }
+    if (result.hedged) this.stats.hedged++
+    if (result.secondWon) this.stats.hedgeWon++
+    return result.value
+  }
+
+  /** The longest-running request: range, chunk, how long, how much came in. */
+  private oldest() {
+    let oldest: { rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
+    for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
+    if (!oldest) return 'none in flight'
+    return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
+  }
+
+  private sample() {
+    const s = this.stats
+    s.samples++
+    s.fetching += this.fetchingNow
+    s.writing += this.writingNow
+    s.idle += Math.max(0, this.loops - this.fetchingNow - this.writingNow)
+    s.inFlight += NntpPool.busyCount
+    s.inFlightMin = Math.min(s.inFlightMin, NntpPool.busyCount)
+    s.ranges += this.perRange.size
+    s.groups += this.perGroup.size
+  }
+
   /** A line a minute: what the connections fetched, and how long a chunk took. */
   private report() {
     const seconds = (Date.now() - this.stats.since) / 1000
     if (seconds < 60) return
-    const { chunks, articles, fetchMs } = this.stats
+    const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
     if (chunks > 0) {
+      const ages = byAge
+        .map((b, i) => `${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace('– min', '+ min'))
+        .join(', ')
       logger.info(
         `Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
-          `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk, connections busy ${Math.round((fetchMs / (seconds * 1000 * NntpPool.size)) * 100)}%.`
+          `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
+          `hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
+          (this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus` : '') +
+          (this.stats.captured > 0 ? `; ${this.stats.captured} fetched into the corpus only` : '') +
+          '.'
+      )
+      const st = this.stats
+      const avg = (n: number) => (samples > 0 ? n / samples : 0)
+      const per = (ms: number) => (requests > 0 ? (ms / requests / 1000).toFixed(1) : '-')
+      logger.info(
+        `Connections: ${avg(st.inFlight).toFixed(1)} of ${NntpPool.size} with a command in flight (min ${st.inFlightMin}), ` +
+          `on ${avg(st.ranges).toFixed(1)} ranges of ${avg(st.groups).toFixed(1)} groups; ` +
+          `loops ${avg(st.fetching).toFixed(1)} fetching, ${avg(st.writing).toFixed(1)} writing, ${avg(st.idle).toFixed(1)} idle; ` +
+          `a request: GROUP ${per(st.groupMs)} s, XOVER reply ${per(st.replyMs)} s, transfer ${per(st.bodyMs)} s; ` +
+          `${st.silent} of ${requests} transfers silent >5 s; writing ${(st.writeMs / Math.max(1, chunks)).toFixed(0)} ms a chunk; ` +
+          `${st.deadlines} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB a minute); ${this.oldest()}.`
       )
     }
-    this.stats = { chunks: 0, articles: 0, fetchMs: 0, since: Date.now() }
+    this.stats = this.freshStats()
   }
 
   private peek(): boolean {
-    return this.targets.some((t) => t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
+    return this.targets.some((t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
   }
 
-  /** The first unclaimed missing chunk, in target order. */
+  /** Whether the target is under its range's and its group's limit of connections. */
+  private open(target: Target) {
+    const { perRange, perGroup } = coordinatorConfig.spool
+    return (perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) && (perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
+  }
+
+  private count(target: Target, by: number) {
+    const range = (this.perRange.get(target.id) ?? 0) + by
+    const group = (this.perGroup.get(target.group) ?? 0) + by
+    if (range > 0) this.perRange.set(target.id, range)
+    else this.perRange.delete(target.id)
+    if (group > 0) this.perGroup.set(target.group, group)
+    else this.perGroup.delete(target.group)
+  }
+
+  /** The first unclaimed missing chunk, in target order, of a target under its limits. */
   private take(): { target: Target; span: ChunkSpan } | null {
     for (const target of this.targets) {
+      if (!this.open(target)) continue
       const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
       if (index === -1) continue
       const [span] = target.missing.splice(index, 1)
       this.claimed.add(`${target.id}:${span.from}`)
+      this.count(target, 1)
       return { target, span }
     }
     return null
@@ -134,7 +373,7 @@ export class Prefetcher {
       this.spoolBytes = await Spool.size()
     }
     const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
-    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id'))
+    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
     // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
     const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
     // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
@@ -142,8 +381,16 @@ export class Prefetcher {
     const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
     const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
     const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
-    const fresh = room && Number(ready) < ahead
+    // With at most perRange connections a range, enough ranges still being fetched for all loops: fully
-      ? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(ahead - Number(ready))
+    // spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
+    // 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
+    const { perRange } = coordinatorConfig.spool
+    const short = perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
+    const want = Math.max(ahead - Number(ready), short)
+    // The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
+    const full = await CatalogSpace.full()
+    const fresh = room && want > 0
+      ? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').where((q: any) => (full ? q.where('r.priority', '<', 0) : q)).orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(want)
       : []
     const targets: Target[] = []
     for (const row of [...leased, ...(room ? started : []), ...fresh]) {
@@ -154,7 +401,38 @@ export class Prefetcher {
       for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
         if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
       }
-      if (missing.length > 0) targets.push({ id: Number(row.id), group: row.group, missing, total })
+      if (missing.length > 0) {
+        targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
+      }
+    }
+    // Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
+    for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id)))) targets.push(target)
+    return targets
+  }
+
+  /** Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus. */
+  private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
+    const { dir, ahead } = coordinatorConfig.corpus
+    if (!dir || ahead <= 0) return []
+    const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
+    const targets: Target[] = []
+    for (const row of candidates) {
+      if (targets.length >= ahead) break
+      const id = Number(row.id)
+      if (taken.has(id) || this.stored.has(id)) continue
+      const start = Number(row.start_id)
+      const end = Number(row.end_id)
+      const range = { group: row.group, start, end }
+      if (await Corpus.complete(range)) {
+        this.stored.add(id)
+        continue
+      }
+      const chunks = Spool.chunksOf({ start, end })
+      const missing: ChunkSpan[] = []
+      for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
+      if (missing.length > 0) {
+        targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
+      }
     }
     return targets
   }

app/services/prober.ts

@@ -0,0 +1,297 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { setTimeout as sleep } from 'node:timers/promises'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { YencError } from '#lib/yenc/index'
+import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
+
+interface Candidate {
+  id: number
+  releaseId: number
+  kind: string
+  messageId: string
+}
+
+type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
+
+/**
+ * The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
+ * par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones).
+ */
+const TIERS: { kind: string; unreadable: boolean }[] = [
+  { kind: 'lone', unreadable: false },
+  { kind: 'par2', unreadable: true },
+  { kind: 'archive', unreadable: true },
+  { kind: 'par2', unreadable: false },
+  { kind: 'archive', unreadable: false },
+]
+const FLUSH_MS = 2000
+/** Slices a claim takes from random points of the queue. */
+const SLICES = 20
+const REPORT_MS = 60_000
+
+/**
+ * Phase 2's first step (REBUILD.md, part 6): reads the probe queue's
+ * articles (settled releases first, by kind) over PROBE_CONNECTIONS
+ * connections of its own, without compression, and stores what they show:
+ * per release and kind an Inspection, and on the release whether it's
+ * passworded and its inner name. One process: fetching (~0.1 s an article)
+ * dwarfs parsing (a few ms), so there's no queue between them.
+ */
+export class Prober {
+  private queue: Candidate[] = []
+  private outcomes: Outcome[] = []
+  private refilling: Promise<void> | null = null
+  private stopped = false
+  private stats = Prober.freshStats()
+
+  /** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
+  private static freshStats() {
+    return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
+  }
+
+  async run() {
+    // A previous run's claims: one prober at a time.
+    const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
+    if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
+    const connections = coordinatorConfig.probe.connections
+    logger.info(`Probing with ${connections} connections.`)
+    const flusher = (async () => {
+      while (!this.stopped || this.outcomes.length > 0) {
+        await sleep(FLUSH_MS)
+        const flushed = Date.now()
+        await this.flush().catch((error) => logger.error(`Probing: storing results failed: ${error.message}`))
+        this.stats.flushMs += Date.now() - flushed
+        this.report()
+      }
+    })()
+    await Promise.all(Array.from({ length: connections }, () => this.lane()))
+    // What was claimed and not probed goes back.
+    for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
+    this.stopped = true
+    await flusher
+  }
+
+  stop() {
+    this.stopped = true
+  }
+
+  /** One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure. */
+  private async lane() {
+    let client: XoverClient | null = null
+    while (!this.stopped) {
+      const batch: Candidate[] = []
+      const waited = Date.now()
+      while (batch.length < coordinatorConfig.probe.pipeline) {
+        const candidate = await this.next()
+        if (!candidate) break
+        batch.push(candidate)
+      }
+      this.stats.waitMs += Date.now() - waited
+      if (batch.length === 0) {
+        this.stats.idleMs += 30_000
+        await sleep(30_000)
+        continue
+      }
+      try {
+        client ??= await XoverClient.connect({
+          host: env.get('NNTP_HOST')!,
+          port: env.get('NNTP_PORT', 563),
+          user: env.get('NNTP_USER'),
+          password: env.get('NNTP_PASSWORD'),
+          secure: env.get('NNTP_SECURE', true),
+          compress: false,
+          timeoutMs: coordinatorConfig.nntpTimeoutMs,
+        })
+        const fetched = Date.now()
+        const bodies = await client.bodies(batch.map((c) => c.messageId))
+        this.stats.fetchMs += Date.now() - fetched
+        const inspected = Date.now()
+        for (const [i, candidate] of batch.entries()) {
+          const body = bodies[i]
+          this.stats.articles++
+          if (body === null) {
+            // Taken down or expired: final.
+            this.outcomes.push({ candidate, status: 'missing' })
+            continue
+          }
+          this.stats.bytes += body.length
+          try {
+            this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) })
+          } catch (error: any) {
+            // Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
+            if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
+            this.outcomes.push({ candidate, status: 'error' })
+          }
+        }
+        this.stats.inspectMs += Date.now() - inspected
+      } catch (error: any) {
+        logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
+        for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
+        client?.close()
+        client = null
+        await sleep(5000)
+      }
+    }
+    client?.close()
+  }
+
+  private async next(): Promise<Candidate | undefined> {
+    if (this.queue.length < coordinatorConfig.probe.batch / 2) {
+      this.refilling ??= this.claim().finally(() => (this.refilling = null))
+      if (this.queue.length === 0) await this.refilling
+    }
+    return this.queue.shift()
+  }
+
+  /**
+   * Waiting candidates of settled releases, by kind, claimed so another round
+   * doesn't take them: SLICES slices from random points of the queue. In id
+   * order it went group by group, and some groups' articles take 0.6–3 s
+   * where most take 20–60 ms (a.b.zogg 2016: 7 articles/s on 6 connections).
+   */
+  private async claim() {
+    const started = Date.now()
+    try {
+      await this.claimTiers()
+    } finally {
+      this.stats.claimMs += Date.now() - started
+    }
+  }
+
+  private async claimTiers() {
+    for (const { kind, unreadable } of TIERS) {
+      const want = coordinatorConfig.probe.batch - this.queue.length
+      if (want <= 0) return
+      // The starting points chosen here: in the query Postgres drew random() once for all slices.
+      // Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
+      const [{ lo, hi }] = (
+        await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable >= ?`, [kind, unreadable])
+      ).rows
+      if (lo === null) continue
+      const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
+      const { rows } = await db.rawQuery(
+        `UPDATE probe_candidates SET status = 'taken' WHERE id IN (
+           SELECT pc.id FROM unnest(?::bigint[]) AS o(start)
+           CROSS JOIN LATERAL (
+             SELECT pc.id FROM probe_candidates pc
+             WHERE pc.status = 'waiting' AND pc.ready AND pc.kind = ? AND pc.unreadable >= ? AND pc.id >= o.start
+             ORDER BY pc.kind, pc.unreadable, pc.id LIMIT ? FOR UPDATE SKIP LOCKED) pc)
+         RETURNING id, release_id, kind, message_id`,
+        [starts, kind, unreadable, Math.ceil(want / SLICES)]
+      )
+      const claimed = rows.map((r: any) => ({ id: Number(r.id), releaseId: Number(r.release_id), kind: r.kind, messageId: r.message_id }))
+      // Probed before (an earlier catalog's candidate of the same article): answered from probe_cache.
+      const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
+        claimed.map((c: Candidate) => c.messageId),
+        PROBE_VERSION,
+      ])
+      const known = new Map(cached.map((c: any) => [c.message_id, c]))
+      for (const candidate of claimed) {
+        const hit: any = known.get(candidate.messageId)
+        if (hit) {
+          this.stats.cached++
+          this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
+        } else {
+          this.queue.push(candidate)
+        }
+      }
+    }
+  }
+
+  /** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
+  private async flush() {
+    const outcomes = this.outcomes.splice(0)
+    if (outcomes.length === 0) return
+    for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
+    await db.transaction(async (trx) => {
+      await trx.rawQuery(
+        `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
+        [JSON.stringify(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
+      )
+      // Final answers read from the news server, kept by message-id for the next catalog (a reset).
+      const fresh = outcomes.filter((o) => !o.cached && o.status !== 'waiting')
+      if (fresh.length > 0) {
+        await trx.rawQuery(
+          `INSERT INTO probe_cache (message_id, status, result, version)
+           SELECT v.message_id, v.status, v.result, ? FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, status text, result jsonb)
+           ON CONFLICT (message_id) DO UPDATE SET status = excluded.status, result = excluded.result, version = excluded.version, created_at = now()`,
+          [PROBE_VERSION, JSON.stringify(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
+        )
+      }
+      const done = outcomes.filter((o) => o.inspection)
+      if (done.length === 0) return
+      await trx.rawQuery(
+        `INSERT INTO release_inspections (release_id, candidate_id, kind, result)
+         SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
+         ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
+        // A par2 set's file list goes to par2_files, not into every inspection.
+        [JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
+      )
+      await this.storeForMerge(trx, done)
+      // Each touched release from all its inspections (earlier rounds may have stored others).
+      const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
+      const { rows } = await trx.rawQuery(
+        `SELECT i.release_id, i.kind, i.result, r.name FROM release_inspections i JOIN releases r ON r.id = i.release_id WHERE i.release_id = ANY(?)`,
+        [ids]
+      )
+      const byRelease = new Map<number, { name: string; all: { kind: string; result: Inspection }[] }>()
+      for (const r of rows) {
+        const id = Number(r.release_id)
+        const entry = byRelease.get(id) ?? { name: r.name as string, all: [] as { kind: string; result: Inspection }[] }
+        entry.all.push({ kind: r.kind, result: r.result })
+        byRelease.set(id, entry)
+      }
+      const updates = [...byRelease].map(([id, { name, all }]) => ({
+        id,
+        passworded: all.some((i) => i.result.passworded === true),
+        inner_name: innerName(all, name),
+      }))
+      await trx.rawQuery(
+        `UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
+         FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
+        [JSON.stringify(updates)]
+      )
+    })
+  }
+
+  /** What the par2 merge matches (app/services/par2_merger.ts): files' first-16-KiB hashes, par2 sets' file lists. */
+  private async storeForMerge(trx: any, done: Outcome[]) {
+    const hashed = done.filter((o) => o.inspection!.hash16k)
+    if (hashed.length > 0) {
+      await trx.rawQuery(
+        `INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
+         SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
+         ON CONFLICT (candidate_id) DO NOTHING`,
+        [JSON.stringify(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
+      )
+    }
+    for (const o of done) {
+      const set = o.inspection!.par2
+      if (!set) continue
+      await trx.rawQuery(
+        `INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
+         SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
+         ON CONFLICT DO NOTHING`,
+        [set.setId, JSON.stringify(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
+      )
+      await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
+    }
+  }
+
+  private report() {
+    const seconds = (Date.now() - this.stats.since) / 1000
+    if (seconds < REPORT_MS / 1000) return
+    const { articles, bytes, done, missing, error } = this.stats
+    logger.info(
+      `Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
+        `${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache. ` +
+        `Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
+        `idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
+        `claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`
+    )
+    this.stats = Prober.freshStats()
+  }
+}

app/services/range_blocks.ts

@@ -0,0 +1,31 @@
+import db from '@adonisjs/lucid/services/db'
+
+/**
+ * range_blocks (migration 3900): per group, each run of consecutive done ranges with its posting time span, read by
+ * release_final(). A few thousand rows from ~650k ranges, rebuilt whole in one statement each time; a run breaks at a
+ * range that isn't done or doesn't start where the one before ended.
+ */
+class RangeBlocks {
+  async rebuild(): Promise<number> {
+    return db.transaction(async (trx) => {
+      await trx.rawQuery(`DELETE FROM range_blocks`)
+      const result = await trx.rawQuery(
+        `WITH r AS (
+           SELECT group_id, start_id, end_id, status = 'done' AS done, posted_low, posted_high,
+                  min(start_id) OVER (PARTITION BY group_id) AS group_first,
+                  lag(status = 'done') OVER w AS done_before, lag(end_id) OVER w AS end_before
+           FROM ranges WINDOW w AS (PARTITION BY group_id ORDER BY start_id)),
+         runs AS (
+           SELECT *, sum(CASE WHEN done IS DISTINCT FROM done_before OR end_before IS DISTINCT FROM start_id - 1 THEN 1 ELSE 0 END)
+                       OVER (PARTITION BY group_id ORDER BY start_id) AS run
+           FROM r)
+         INSERT INTO range_blocks (group_id, start_id, end_id, posted_low, posted_high, at_group_start)
+         SELECT group_id, min(start_id), max(end_id), min(posted_low), max(posted_high), min(start_id) = min(group_first)
+         FROM runs WHERE done GROUP BY group_id, run`
+      )
+      return result.rowCount ?? 0
+    })
+  }
+}
+
+export default new RangeBlocks()

app/services/release_judge.ts

@@ -0,0 +1,159 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Raised when a rule changes: verdicts of an older version are judged again. */
+export const JUDGE_VERSION = 1
+/** Between rounds: each rule is a scan of its table. */
+const ROUND_MS = 60 * 60 * 1000
+/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
+const REPEATED_RELEASES = 20
+
+/**
+ * A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. No "?" in
+ * these patterns: a raw query takes it for a binding ("h[ .]?26[45]" and "web[ .-]?dl" never matched; LOG.md
+ * 2026-10-16 (64)), so {0,1}.
+ */
+const MEDIA =
+  `(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]{0,1}26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]{0,1}dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
+/** The posters of scraped celebrity pictures. */
+const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
+/** An executable's name. */
+const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
+
+export interface Rule {
+  name: string
+  /** Why, in a few words (the log, the web interface). */
+  says: string
+  /** SELECT of the release ids the rule judges. */
+  sql: string
+}
+
+/**
+ * The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
+ * the header pass, the prober, the merges and the spots know by now.
+ */
+export const RULES: Rule[] = [
+  {
+    name: 'dead',
+    says: 'every probed first article is gone from the server (taken down or expired)',
+    sql: `SELECT release_id FROM probe_candidates WHERE status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(status = 'missing')`,
+  },
+  {
+    name: 'fake',
+    says: 'named as a film, episode or music, but the content is a program',
+    // An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
+    // an executable by name or by its first bytes, under a media name.
+    sql: `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
+          WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
+            AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
+            AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
+          UNION
+          SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
+          WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
+            AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
+  },
+  {
+    name: 'repeated',
+    says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
+    sql: `SELECT DISTINCT h.release_id FROM file_hashes h
+          JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
+  },
+  {
+    name: 'scraped',
+    says: 'celebrity pictures scraped from the web, reposted by one spam family',
+    // "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
+    // a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
+    // titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
+    sql: `SELECT r.id FROM releases r
+          WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
+  },
+  {
+    name: 'locked',
+    says: 'passworded, and no spot, NZB or title gives the password',
+    sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
+  },
+  {
+    name: 'unusable',
+    says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
+    // Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
+    sql: `SELECT r.id FROM releases r
+          WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
+            AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
+            AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
+            AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
+            AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
+  },
+]
+
+export interface JudgeRound {
+  /** Per rule: releases it judges now, verdicts added, verdicts taken back. */
+  rules: Record<string, { matched: number; added: number; removed: number }>
+}
+
+/**
+ * The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
+ * signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Each
+ * round recomputes every rule: a release a rule matches gets a verdict (the first rule in order, over a later one's), one it no longer
+ * matches loses its verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
+ */
+export class ReleaseJudge {
+  private stopped = false
+
+  stop() {
+    this.stopped = true
+  }
+
+  async run() {
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        const round = await this.round()
+        const parts = Object.entries(round.rules).map(([n, r]) => `${n} ${r.matched} (+${r.added} −${r.removed})`)
+        logger.info(`Judged: ${parts.join(', ')}.`)
+      } catch (error: any) {
+        logger.error(`Judge round failed: ${error.message}`)
+      }
+      for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  /** One round over every rule; with dry, only counts. */
+  async round(dry = false): Promise<JudgeRound> {
+    const round: JudgeRound = { rules: {} }
+    // Verdicts of an older rule set are judged again from scratch.
+    if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
+    const judged = new Set<string>()
+    for (const rule of RULES) {
+      if (this.stopped) break
+      const { rows } = await db.rawQuery(rule.sql)
+      const ids = rows.map((r: Record<string, unknown>) => String(Object.values(r)[0])).filter((id: string) => !judged.has(id))
+      for (const id of ids) judged.add(id)
+      const stats = { matched: ids.length, added: 0, removed: 0 }
+      if (!dry) {
+        await db.transaction(async (trx) => {
+          // Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
+          await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
+          await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
+          for (let i = 0; i < ids.length; i += 50_000) {
+            await trx.rawQuery(`INSERT INTO judged SELECT unnest(?::bigint[]) ON CONFLICT DO NOTHING`, [ids.slice(i, i + 50_000)])
+          }
+          const removed = await trx.rawQuery(
+            `DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
+            [rule.name]
+          )
+          const added = await trx.rawQuery(
+            `INSERT INTO release_verdicts (release_id, rule, rule_version)
+             SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
+             ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
+             WHERE release_verdicts.rule <> excluded.rule`,
+            [rule.name, JUDGE_VERSION]
+          )
+          stats.removed = removed.rowCount ?? 0
+          stats.added = added.rowCount ?? 0
+        })
+      }
+      round.rules[rule.name] = stats
+    }
+    return round
+  }
+}

app/services/search_service.ts

@@ -0,0 +1,317 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import NzbService from '#services/nzb_service'
+
+export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
+
+/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
+export interface SearchParams {
+  q?: string
+  /** Also releases the junk judge hid (release_verdicts). */
+  judged?: boolean
+  /** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
+  readable?: boolean
+  /** Only releases known to have all their files. */
+  complete?: boolean
+  /** Leave out releases known to miss files (Newznab). */
+  notIncomplete?: boolean
+  /** Leave out releases the prober found passworded (encrypted archive entries or headers). */
+  unlocked?: boolean
+  /** How the release was recognised: stem, subject or post. */
+  source?: string
+  poster?: string
+  /** Exact category ids (already expanded from parents). */
+  categories?: number[]
+  maxAgeDays?: number
+  /** TV: season and episode, matched as S01E02 or 1x02 in the name. */
+  season?: number
+  episode?: number
+  /** Size bounds in bytes. */
+  minSize?: number
+  maxSize?: number
+  sort?: SortOrder
+  page?: number
+  perPage?: number
+  /** Overrides `page` (Newznab pages by offset). */
+  offset?: number
+  /** Count matches up to this many past the page; `total` is capped there. */
+  countLimit?: number
+}
+
+/** A release in search results. */
+export interface ReleaseSummary {
+  id: number
+  name: string
+  source: string
+  poster: string | null
+  groups: string[]
+  postedAt: Date | null
+  fileTotal: number | null
+  fileCount: number
+  size: number
+  readable: boolean
+  complete: boolean | null
+  /** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
+  final: boolean
+  category: number
+  settled: boolean
+  /** The prober found encrypted archive entries or headers; null: not probed (yet). */
+  passworded: boolean | null
+  /** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
+  innerName: string | null
+}
+
+/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
+const MAX_POSTERS = 200
+const SORTS: Record<SortOrder, string> = {
+  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
+  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
+  largest: 'r.bytes DESC, r.id DESC',
+  files: 'r.files DESC, r.id DESC',
+  name: 'r.name ASC, r.id ASC',
+}
+
+/** A broad search needn't count every match (each counted row is a heap read). */
+const COUNT_LIMIT = 10_000
+const STATS_TTL_MS = 5 * 60 * 1000
+/** A refresh that takes longer is given up (the next request tries again). */
+const STATS_TIMEOUT_MS = 2 * 60 * 1000
+
+/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
+function contains(word: string): string {
+  return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
+}
+
+/** Searching and browsing the catalog, for the web interface and Newznab. */
+class SearchService {
+  async search(params: SearchParams) {
+    const where: string[] = ['r.files > 0']
+    // Judged junk (app/services/release_judge.ts) is left out unless asked for.
+    if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
+    const bindings: unknown[] = []
+    // Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
+    for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
+      where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
+      bindings.push(contains(word), contains(word))
+    }
+    if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
+    if (params.complete) where.push('r.complete')
+    // Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
+    if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
+    if (params.unlocked) where.push('r.passworded IS NOT TRUE')
+    if (params.source) {
+      where.push('r.source = ?')
+      bindings.push(params.source)
+    }
+    // A poster search: the matching posters first (posters, by trigram), each one's releases then by
+    // releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
+    // poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
+    let posters: string[] | null = null
+    if (params.poster) {
+      const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
+      posters = matched.map((m: { poster: string }) => m.poster)
+    }
+    if (params.categories?.length) {
+      where.push('r.category = ANY(?)')
+      bindings.push(params.categories)
+    }
+    if (params.maxAgeDays) {
+      where.push('r.posted_at >= now() - make_interval(days => ?)')
+      bindings.push(params.maxAgeDays)
+    }
+    if (params.minSize !== undefined) {
+      where.push('r.bytes >= ?')
+      bindings.push(params.minSize)
+    }
+    if (params.maxSize !== undefined) {
+      where.push('r.bytes <= ?')
+      bindings.push(params.maxSize)
+    }
+    if (params.season !== undefined) {
+      const s = params.season
+      const e = params.episode
+      where.push('r.name ~* ?')
+      bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
+    }
+
+    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
+    const page = Math.max(params.page ?? 1, 1)
+    const offset = params.offset ?? (page - 1) * perPage
+    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
+    const whereSql = where.join(' AND ')
+    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
+    if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
+    // Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
+    // bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
+    const source =
+      posters === null
+        ? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
+        : {
+            from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
+                     SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
+            where: 'true',
+            bindings: [posters] as unknown[],
+          }
+    const counting =
+      posters === null
+        ? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
+        : db.rawQuery(
+            `SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
+               SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
+            [posters, ...bindings]
+          )
+    const [{ rows }, { rows: counted }] = await Promise.all([
+      db.rawQuery(
+        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
+                release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
+         FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
+        [...source.bindings, ...bindings, perPage, offset]
+      ),
+      counting,
+    ])
+    return {
+      page,
+      perPage,
+      offset,
+      total: Math.min(Number(counted[0].total), countLimit),
+      /** More than `total` match (the count stops at `countLimit`). */
+      totalCapped: Number(counted[0].total) > countLimit,
+      releases: (rows as any[]).map((r): ReleaseSummary => ({
+        id: Number(r.id),
+        name: r.name,
+        source: r.source,
+        poster: r.poster,
+        groups: r.groups,
+        postedAt: r.posted_at,
+        fileTotal: r.file_total,
+        fileCount: Number(r.files),
+        size: Number(r.bytes),
+        readable: r.readable,
+        complete: r.complete,
+        final: r.final,
+        category: r.category,
+        settled: r.settled_at !== null,
+        passworded: r.passworded,
+        innerName: r.inner_name,
+      })),
+    }
+  }
+
+  /** A release with its files, read from its chunks. */
+  async release(id: number) {
+    const r = await db.from('releases').where('id', id).first()
+    if (!r) return null
+    const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
+      NzbService.files(id),
+      db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
+      db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
+    ])
+    return {
+      id: Number(r.id),
+      name: r.name,
+      source: r.source,
+      poster: r.poster,
+      groups: r.groups,
+      postedAt: r.posted_at,
+      fileTotal: r.file_total,
+      fileCount: Number(r.files),
+      size: Number(r.bytes),
+      segments: Number(r.segments),
+      readable: r.readable,
+      complete: r.complete,
+      final: finality[0].final,
+      hasPar2: r.has_par2,
+      category: r.category,
+      firstSeenAt: r.first_seen_at,
+      lastSeenAt: r.last_seen_at,
+      settledAt: r.settled_at,
+      passworded: r.passworded,
+      innerName: r.inner_name,
+      inspectedAt: r.inspected_at,
+      /** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
+      inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
+      files: files.map((f) => ({
+        name: f.name,
+        subject: f.subject,
+        poster: f.poster,
+        postedAt: f.postedAt === null ? null : new Date(f.postedAt),
+        size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
+        parts: f.segments.length,
+      })),
+    }
+  }
+
+  /** Totals for the web interface, computed at most once a minute (callers in between share them). */
+  async stats() {
+    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
+      this.statsRefresh ??= Promise.race([
+        this.computeStats(),
+        new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()),
+      ])
+        .then((value) => {
+          this.statsCache = { at: Date.now(), value }
+        })
+        .catch((error) => {
+          logger.warn(`Stats: ${error.message}`)
+          if (!this.statsCache) throw error
+        })
+        .finally(() => {
+          this.statsRefresh = undefined
+        })
+      if (!this.statsCache) await this.statsRefresh
+    }
+    return this.statsCache!.value
+  }
+
+  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
+  private statsRefresh?: Promise<void>
+
+  /**
+   * Estimates (a catalog of 100M+ rows can't be counted on every visit):
+   * the row count from Postgres's statistics, below a million rows exact;
+   * readable share, files and size from a sample of 5,000 rows; ranges
+   * counted exactly (a few hundred thousand rows).
+   */
+  private async computeStats() {
+    const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
+    const estimate = Number(rows[0]?.n ?? 0)
+    // A fixed-size sample: a percentage of a big catalog is many random reads on the HDD.
+    const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)'
+    const { rows: s } = await db.rawQuery(
+      `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
+       FROM releases ${sample} WHERE files > 0`
+    )
+    const sampled = Number(s[0].sampled)
+    const releases = sample ? Math.max(estimate, 0) : sampled
+    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
+    // History by articles; the pass: the history ranges before those deferred behind them (priority
+    // 2000 apart; e.g. groups added later that go first).
+    const { rows: ranges } = await db.rawQuery(
+      `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
+            pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
+       SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
+              coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
+              coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
+              coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
+              (SELECT max(done_at) FROM ranges) AS last_done_at,
+              (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
+       FROM h`
+    )
+    const g = ranges[0]
+    return {
+      approximate: sample !== '',
+      releases,
+      readableReleases: scale(s[0].readable),
+      files: scale(s[0].files),
+      size: scale(s[0].bytes),
+      ranges: { done: Number(g.done), total: Number(g.total) },
+      /** History and the current pass, in articles. */
+      history: { done: Number(g.history_done), total: Number(g.history) },
+      pass: { done: Number(g.pass_done), total: Number(g.pass) },
+      lastIndexedAt: g.last_done_at,
+      tailIndexedAt: g.tail_done_at,
+    }
+  }
+}
+
+export default new SearchService()

app/services/settler.ts

@@ -2,26 +2,45 @@ import db from '@adonisjs/lucid/services/db'
 
 /** Releases unchanged for this long are settled (ranges are read in order; the merger adds within minutes). */
 const QUIET_HOURS = 6
+/**
+ * History: a release whose last post is QUIET_HOURS old gets nothing new from the tail, only from the
+ * neighbouring ranges' fragments, which the merger adds within minutes: settled once unchanged this long.
+ * Waiting QUIET_HOURS after it was read held every history release back from phase 2 for six hours after
+ * each reset (the hourly sweeps' resets kept the prober idle; LOG.md 2026-10-12 (13)).
+ */
+const QUIET_HISTORY_MINUTES = 30
 const BATCH = 50_000
 
 /**
  * Settles releases (REBUILD.md, part 8): once nothing has been added for
- * QUIET_HOURS, its completeness is decided: complete when every file the
+ * QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
  * [i/N] counter announced was found, unknown without a counter (phase 2
  * reads par2 sets for those). Phase 2 picks its work from settled releases.
  */
 class Settler {
-  async run(): Promise<number> {
+  /** Up to `batches` batches of BATCH (all there are without). */
+  async run(batches = Infinity): Promise<number> {
     let settled = 0
-    while (true) {
+    for (let batch = 0; batch < batches; batch++) {
-      const { rowCount } = await db.rawQuery(
+      // Their probe candidates become claimable (Prober.claim), the unreadable ones first.
-        `UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
+      const { rows } = await db.rawQuery(
-         WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(hours => ?) LIMIT ?)`,
+        `WITH settled AS (
-        [QUIET_HOURS, BATCH]
+           UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
+           WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
+             AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
+           RETURNING id, readable, files, bytes, poster),
+         marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
+         -- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them.
+         queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
+           ON CONFLICT DO NOTHING)
+         SELECT count(*)::int AS n FROM settled`,
+        [QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
       )
-      settled += rowCount ?? 0
+      const n = rows[0].n
-      if ((rowCount ?? 0) < BATCH) return settled
+      settled += n
+      if (n < BATCH) return settled
     }
+    return settled
   }
 }
 

app/services/sidecar_folder.ts

@@ -0,0 +1,91 @@
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
+
+/** Candidates looked at per round. */
+const BATCH = 2000
+/** An upload's sidecar is posted with it: its release is looked for this far from it. */
+const WINDOW_MINUTES = 10
+/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
+const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
+
+/** The words of a name worth comparing: 4+ letters or digits. */
+function words(text: string): Set<string> {
+  return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4))
+}
+
+export interface Sibling {
+  id: number
+  name: string
+  files: number
+  file_total: number | null
+}
+
+/**
+ * Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject
+ * ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one
+ * under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that
+ * isn't one release.
+ */
+export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null {
+  const said = words(files.map((f) => f.subject).join(' '))
+  const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w)))
+  if (named.length === 1) return named[0]
+  if (named.length > 1) return null
+  const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : []
+  return counted.length === 1 ? counted[0] : null
+}
+
+export interface FoldRound {
+  looked: number
+  sidecars: number
+  folded: number
+}
+
+/**
+ * The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another
+ * counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small
+ * settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in
+ * the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via
+ * 'sidecar'). The rest leave the queue untouched.
+ */
+export class SidecarFolder {
+  constructor(private merge: (trx: any, found: Map<string, any>, key: string, via: 'sidecar') => Promise<number>) {}
+
+  async round(): Promise<FoldRound> {
+    const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
+    const { rows: queued } = await db.rawQuery(
+      `SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
+       FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
+      [BATCH]
+    )
+    const { rows: stale } = await db.rawQuery(
+      `SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
+      [BATCH]
+    )
+    const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
+    for (const c of queued) {
+      round.looked++
+      done.push(Number(c.id))
+      const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
+      const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
+      if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
+      round.sidecars++
+      if (!c.posted_at) continue
+      const { rows: siblings } = await db.rawQuery(
+        `SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
+         WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
+           AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`,
+        [c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files]
+      )
+      const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
+      if (!sibling) continue
+      const full = siblings.find((s: any) => Number(s.id) === sibling.id)
+      const merged = await db.transaction((trx) =>
+        this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar')
+      )
+      if (merged > 0) round.folded++
+    }
+    if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete()
+    return round
+  }
+}

app/services/spot_inspector.ts

@@ -0,0 +1,359 @@
+import { createHash } from 'node:crypto'
+import { gzipSync } from 'node:zlib'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { decodeSpotBinary } from '#lib/spotnet/index'
+import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
+import { parseSubject } from '#lib/headers/index'
+import { releaseIdentity } from '#lib/release/index'
+
+/** Spots claimed per round, newest first. */
+const BATCH = 20
+/** Between rounds when there was nothing to do. */
+const IDLE_MS = 60_000
+/** Unmatched spots are matched again this often (the header pass may have built their releases since). */
+const REMATCH_MS = 60 * 60 * 1000
+/**
+ * Unmatched spots matched again per hour: after a reset every spot is (16.5k at 500 an hour took 33 h). Never tried
+ * first, then the newest: the oldest inspection first tried a 3-month backfill before today's spots, the ones whose
+ * releases the tail has (2 of 16.7k matched 3 h after reset 24).
+ */
+const REMATCH_BATCH = 5000
+/** Unmatched spots are matched again for this long after their inspection. */
+const REMATCH_DAYS = 30
+/** A file's release posted further apart than this from the NZB's date is another upload of the same name. */
+const MAX_SPREAD_MS = 48 * 3600 * 1000
+
+export interface InspectRound {
+  inspected: number
+  matched: number
+  standalone: number
+  outcomes: Record<string, number>
+}
+
+interface SpotFile {
+  idx: number
+  name: string
+  subject: string
+  poster: string
+  groups: string[]
+  postedAt: Date | null
+  bytes: number
+  segments: number
+}
+
+/**
+ * A placeholder NZB (Dummy.rar + Dummy.nzb, shared by many spots): under 1% of the size the spot announces, or under
+ * 16 KB in all (text-only spots). Its files aren't listed (voyager 1's isDummyNzb).
+ */
+export function isDummyNzb(nzb: ParsedNzb, spotSize: number): boolean {
+  const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
+  return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
+}
+
+/** A file of an NZB as the header pass names it: the subject's file name, by the same parser. */
+export function nzbFileName(subject: string): { name: string; base: string } {
+  const parsed = parseSubject(subject) ?? parseSubject(`${subject} (1/1)`)
+  return parsed ? { name: parsed.name, base: parsed.base } : { name: subject, base: subject }
+}
+
+/**
+ * The spot inspector (REBUILD.md, spots): looks inside each listed spot, on
+ * one NNTP connection of its own. It reads the spot's compressed NZB (its
+ * articles in alt.binaries.ftd), keeps the list of files in `spot_files`,
+ * and matches every file to the release the header pass built for it: the
+ * release identity of the file's subject and poster (lib/release, as the
+ * edge workers compute it), in one of the file's groups, posted within 48 h.
+ * Each matched release is linked in `release_spots`, and gets the spot's
+ * password in `release_passwords` (the NZB's own `<meta type="password">`
+ * first, else what the spot's title or text names) and `passworded`. A spot
+ * whose files match nothing is matched again every hour, for 30 days; once
+ * SPOT_STANDALONE_AFTER_HOURS have passed since its inspection it becomes a
+ * release of its own (source `spot`) with its NZB, until the header pass
+ * builds the releases after all (a spot older than what the catalog has
+ * indexed): then it moves to those, and its own release goes.
+ */
+export class SpotInspector {
+  private stopped = false
+  private client?: XoverClient
+  private groupIds = new Map<string, number | null>()
+  private rematchedAt = 0
+
+  stop() {
+    this.stopped = true
+    this.client?.close()
+  }
+
+  async run() {
+    while (!this.stopped) {
+      let round: InspectRound | null = null
+      try {
+        round = await this.round()
+        if (round.inspected > 0 || round.standalone > 0) {
+          const outcomes = Object.entries(round.outcomes).map(([k, n]) => `${n} ${k}`).join(', ')
+          logger.info(`Spots inspected: ${round.inspected} (${outcomes}); ${round.matched} matched releases, ${round.standalone} of their own.`)
+        }
+      } catch (error: any) {
+        logger.error(`Spot inspection failed: ${error.message}`)
+        this.client?.close()
+        this.client = undefined
+      }
+      if (!round || round.inspected === 0) {
+        for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+      }
+    }
+  }
+
+  private async connect(): Promise<XoverClient> {
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: false,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    return this.client
+  }
+
+  async round(): Promise<InspectRound> {
+    const round: InspectRound = { inspected: 0, matched: 0, standalone: 0, outcomes: {} }
+    const spots = await db.from('spots').where('status', 'listed').orderBy('posted_at', 'desc').limit(BATCH)
+    for (const spot of spots) {
+      if (this.stopped) break
+      const outcome = await this.inspect(spot)
+      round.inspected++
+      round.outcomes[outcome.status] = (round.outcomes[outcome.status] ?? 0) + 1
+      round.matched += outcome.matched
+    }
+    if (Date.now() - this.rematchedAt >= REMATCH_MS && !this.stopped) {
+      this.rematchedAt = Date.now()
+      const again = await this.rematch()
+      round.matched += again.matched
+      round.standalone += again.standalone
+    }
+    return round
+  }
+
+  /** One spot: its NZB read, its files listed and matched. */
+  async inspect(spot: any): Promise<{ status: string; matched: number }> {
+    const done = async (status: string, extra: Record<string, unknown> = {}) => {
+      await db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
+      return { status, matched: 0 }
+    }
+    const segments: string[] = spot.nzb_segments ?? []
+    if (segments.length === 0) return done('no-nzb')
+    if (segments.length > coordinatorConfig.spots.maxNzbSegments) return done('no-nzb', { error: `${segments.length} NZB articles` })
+    const client = await this.connect()
+    const bodies = await client.bodies(segments)
+    if (bodies.some((b) => b === null)) return done('nzb-gone', { error: `${bodies.filter((b) => b === null).length} of ${segments.length} NZB articles gone` })
+    let xml: string
+    try {
+      xml = decodeSpotBinary(bodies as Buffer[]).toString('utf8')
+    } catch (error: any) {
+      return done('nzb-unreadable', { error: String(error.message).slice(0, 500) })
+    }
+    return this.inspectNzb(spot, xml)
+  }
+
+  /** The files of a spot's NZB listed and matched (the tests feed this directly). */
+  async inspectNzb(spot: any, xml: string): Promise<{ status: string; matched: number }> {
+    const nzb = parseNzb(xml)
+    const finish = (status: string, extra: Record<string, unknown> = {}) =>
+      db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
+    if (!nzb || nzb.files.length === 0) {
+      await finish('no-nzb')
+      return { status: 'no-nzb', matched: 0 }
+    }
+    if (isDummyNzb(nzb, Number(spot.size ?? 0))) {
+      await finish('dummy')
+      return { status: 'dummy', matched: 0 }
+    }
+    const files: SpotFile[] = nzb.files.map((f, idx) => ({
+      idx,
+      name: nzbFileName(f.subject).name.slice(0, 1000),
+      subject: f.subject.slice(0, 2000),
+      poster: f.poster,
+      groups: f.groups,
+      postedAt: f.date ? new Date(f.date * 1000) : null,
+      bytes: f.segments.reduce((s, x) => s + x.bytes, 0),
+      segments: f.segments.length,
+    }))
+    // The NZB's own password over what the spot's text names.
+    const password = nzb.meta.password ? { password: nzb.meta.password.slice(0, 200), source: 'nzb' } : spot.password ? { password: spot.password, source: spot.password_source } : null
+    await db.transaction(async (trx) => {
+      await trx.from('spot_files').where('spot_id', spot.message_id).delete()
+      for (let i = 0; i < files.length; i += 1000) {
+        await trx.table('spot_files').multiInsert(
+          files.slice(i, i + 1000).map((f) => ({
+            spot_id: spot.message_id,
+            idx: f.idx,
+            name: f.name,
+            subject: f.subject,
+            poster: f.poster,
+            groups: f.groups,
+            posted_at: f.postedAt,
+            bytes: f.bytes,
+            segments: f.segments,
+          }))
+        )
+      }
+      await trx
+        .from('spots')
+        .where('message_id', spot.message_id)
+        .update({
+          status: 'inspected',
+          inspected_at: new Date(),
+          files: files.length,
+          matched: 0,
+          nzb: gzipSync(xml),
+          ...(password && password.source === 'nzb' ? { password: password.password, password_source: 'nzb' } : {}),
+        })
+    })
+    const matched = await this.match(spot.message_id)
+    return { status: 'inspected', matched }
+  }
+
+  private async groupId(name: string): Promise<number | null> {
+    if (!this.groupIds.has(name)) {
+      const row = await db.from('groups').where('name', name).select('id').first()
+      this.groupIds.set(name, row ? Number(row.id) : null)
+    }
+    return this.groupIds.get(name)!
+  }
+
+  /**
+   * Matches a spot's listed files to releases and links them; how many releases. A file's release: the same
+   * identity (releaseIdentity over its subject and poster) in one of its groups, the one posted nearest the file.
+   */
+  async match(spotId: string): Promise<number> {
+    const spot = await db.from('spots').where('message_id', spotId).first()
+    if (!spot || spot.status !== 'inspected') return 0
+    const files = await db.from('spot_files').where('spot_id', spotId).orderBy('idx')
+    const wanted: { idx: number; key: string; groups: number[]; at: number | null }[] = []
+    for (const f of files) {
+      if (!f.subject) continue
+      const { name, base } = nzbFileName(f.subject)
+      const identity = releaseIdentity({ poster: f.poster ?? '', subject: base, name, weak: false })
+      if (!identity) continue
+      const groups = (await Promise.all((f.groups ?? []).map((g: string) => this.groupId(g)))).filter((g): g is number => g !== null)
+      if (groups.length > 0) wanted.push({ idx: f.idx, key: identity.key, groups, at: f.posted_at ? new Date(f.posted_at).getTime() : null })
+    }
+    if (wanted.length === 0) return 0
+    const { rows } = await db.rawQuery(
+      `SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`,
+      [[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]]
+    )
+    const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
+    for (const r of rows) {
+      const list = byKey.get(r.identity_key) ?? []
+      list.push({ id: Number(r.id), group: Number(r.group_id), at: r.posted_at ? new Date(r.posted_at).getTime() : null })
+      byKey.set(r.identity_key, list)
+    }
+    const fileRelease = new Map<number, number>()
+    for (const w of wanted) {
+      const near = (byKey.get(w.key) ?? [])
+        .filter((r) => w.groups.includes(r.group))
+        .map((r) => ({ ...r, spread: r.at === null || w.at === null ? 0 : Math.abs(r.at - w.at) }))
+        .filter((r) => r.spread <= MAX_SPREAD_MS)
+        .sort((a, b) => a.spread - b.spread)
+      if (near[0]) fileRelease.set(w.idx, near[0].id)
+    }
+    const perRelease = new Map<number, number>()
+    for (const id of fileRelease.values()) perRelease.set(id, (perRelease.get(id) ?? 0) + 1)
+    if (perRelease.size === 0) return 0
+    await db.transaction(async (trx) => {
+      // A release of the spot's own goes now that the header pass built its files' releases.
+      if (spot.release_id) {
+        const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first()
+        if (own) {
+          await trx.from('release_spots').where('release_id', own.id).delete()
+          await trx.from('release_passwords').where('release_id', own.id).delete()
+          await trx.from('releases').where('id', own.id).delete()
+        }
+      }
+      for (const [idx, releaseId] of fileRelease) await trx.from('spot_files').where({ spot_id: spotId, idx }).update({ release_id: releaseId })
+      for (const [releaseId, n] of perRelease) {
+        await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: n }).onConflict(['release_id', 'spot_id']).merge({ files: n })
+      }
+      if (spot.password && perRelease.size > 0) {
+        await trx
+          .table('release_passwords')
+          .multiInsert([...perRelease.keys()].map((id) => ({ release_id: id, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId })))
+          .onConflict(['release_id', 'password'])
+          .ignore()
+        await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
+      }
+      const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
+      // The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
+      await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
+    })
+    return perRelease.size
+  }
+
+  /** Unmatched inspected spots matched again; the ones that waited long enough become releases of their own. */
+  async rematch(): Promise<{ matched: number; standalone: number }> {
+    const out = { matched: 0, standalone: 0 }
+    const { rows: spots } = await db.rawQuery(
+      `SELECT message_id, inspected_at, release_id FROM spots
+       WHERE status = 'inspected' AND matched = 0 AND inspected_at > now() - make_interval(days => ?)
+       ORDER BY rematched_at NULLS FIRST, posted_at DESC NULLS LAST LIMIT ?`,
+      [REMATCH_DAYS, REMATCH_BATCH]
+    )
+    const cutoff = Date.now() - coordinatorConfig.spots.standaloneAfterHours * 3600 * 1000
+    for (const spot of spots) {
+      if (this.stopped) break
+      const n = await this.match(spot.message_id)
+      await db.from('spots').where('message_id', spot.message_id).update({ rematched_at: new Date() })
+      out.matched += n
+      if (n === 0 && !spot.release_id && new Date(spot.inspected_at).getTime() < cutoff && (await this.standalone(spot.message_id))) out.standalone++
+    }
+    return out
+  }
+
+  /** A release of the spot's own (source `spot`), from its listed files; false when it has none. */
+  async standalone(spotId: string): Promise<boolean> {
+    const spot = await db.from('spots').where('message_id', spotId).first()
+    const files = await db.from('spot_files').where('spot_id', spotId)
+    if (!spot || files.length === 0) return false
+    const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
+    const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
+    const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
+    const [row] = await db
+      .table('releases')
+      .insert({
+        release_key: `spot:${key}`,
+        identity_key: key,
+        group_id: groupId,
+        name: spot.title.slice(0, 1000),
+        source: 'spot',
+        poster: spot.poster,
+        groups: JSON.stringify(groups),
+        posted_at: spot.posted_at,
+        last_posted_at: spot.posted_at,
+        files: files.length,
+        segments: files.reduce((s: number, f: any) => s + Number(f.segments), 0),
+        bytes: files.reduce((s: number, f: any) => s + Number(f.bytes), 0),
+        readable: true,
+        passworded: spot.password ? true : null,
+        settled_at: new Date(),
+      })
+      .onConflict('release_key')
+      .ignore()
+      .returning('id')
+    if (!row) return false
+    const releaseId = Number(row.id)
+    await db.transaction(async (trx) => {
+      await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
+      if (spot.password) {
+        await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
+      }
+      await trx.from('spots').where('message_id', spotId).update({ release_id: releaseId })
+    })
+    return true
+  }
+}

app/services/spotter.ts

@@ -0,0 +1,184 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { MissingArticle, XoverClient } from '#lib/nntp/index'
+import {
+  decodeSpotText,
+  moderationTarget,
+  parseSpotArticleHeaders,
+  parseSpotHeader,
+  parseSpotXml,
+  passwordFromSpot,
+  spotterId,
+  verifySpotHeader,
+} from '#lib/spotnet/index'
+
+/** Overviews asked for at a time. */
+const XOVER_STEP = 1000
+/** A personal dispose counts within this long of the spot (Spotweb). */
+const DISPOSE_WINDOW_SECONDS = 5 * 86400
+
+export interface SpotterRound {
+  read: number
+  listed: number
+  unverified: number
+  moderation: number
+}
+
+/**
+ * The spotter (REBUILD.md, spots): Spotnet's spots, from the headers of
+ * free.pt into `spots`, on one NNTP connection of its own. Every few
+ * minutes it reads the overviews after the last article it saw (the first
+ * time the last SPOT_BACKFILL), and for each spot the full header: its
+ * signature is checked there (lib/spotnet, as Spotweb does), and the X-XML
+ * gives the description, the website and the NZB's articles. A password the
+ * title (`{{pw}}`) or the description names is kept with the spot; the spot
+ * inspector carries it to the releases. A delete/dispose message marks its
+ * target `removed` and takes back what the target gave the releases.
+ */
+export class Spotter {
+  private stopped = false
+  private client?: XoverClient
+
+  stop() {
+    this.stopped = true
+    this.client?.close()
+  }
+
+  async run() {
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        const round = await this.round()
+        if (round.read > 0) {
+          logger.info(`Spots: ${round.read} articles read, ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
+        }
+      } catch (error: any) {
+        logger.error(`Spot round failed: ${error.message}`)
+        this.client?.close()
+        this.client = undefined
+      }
+      const wait = coordinatorConfig.spots.everySeconds * 1000
+      for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  private async connect(): Promise<XoverClient> {
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: false,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    return this.client
+  }
+
+  /** The articles after the last one seen, up to the group's high. */
+  async round(): Promise<SpotterRound> {
+    const { group, backfill } = coordinatorConfig.spots
+    const client = await this.connect()
+    const { low, high } = await client.group(group)
+    const state = await db.from('spot_state').where('group_name', group).first()
+    let next = state ? Number(state.last_article) + 1 : Math.max(low, high - backfill + 1)
+    const round: SpotterRound = { read: 0, listed: 0, unverified: 0, moderation: 0 }
+    while (next <= high && !this.stopped) {
+      const to = Math.min(high, next + XOVER_STEP - 1)
+      const overviews = await client.xover(next, to)
+      round.read += overviews.length
+      for (const [number, fields] of overviews) {
+        if (this.stopped) return round
+        const messageId = (fields['message-id'] ?? '').replace(/^<|>$/g, '')
+        if (!messageId || !parseSpotHeader(fields.from ?? '', fields.subject ?? '', messageId, fields.date)) continue
+        const outcome = await this.read(client, messageId, number)
+        if (outcome) round[outcome]++
+      }
+      await db
+        .table('spot_state')
+        .insert({ group_name: group, last_article: to })
+        .onConflict('group_name')
+        .merge({ last_article: to })
+      next = to + 1
+    }
+    return round
+  }
+
+  /** One spot by its full header; what it was, or null when it was there already or is gone. */
+  async read(client: XoverClient, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
+    if (await db.from('spots').where('message_id', messageId).first()) return null
+    let raw: Buffer
+    try {
+      raw = await client.head(messageId)
+    } catch (error) {
+      if (error instanceof MissingArticle) return null
+      throw error
+    }
+    return this.store(raw.toString('latin1'), messageId, articleNumber)
+  }
+
+  /** Verifies and stores a spot from its raw header (the tests feed this directly). */
+  async store(raw: string, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
+    const { headers, xml: xmlText } = parseSpotArticleHeaders(raw)
+    const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
+    if (!header) return null
+    const verified = verifySpotHeader(header)
+    const xml = verified ? parseSpotXml(decodeSpotText(xmlText)) : null
+    const target = verified ? moderationTarget(header) : null
+    const title = xml?.title || header.title
+    // Kept apart by where it came from: the title's {{pw}}, else the description's "Wachtwoord: …".
+    const fromTitle = verified && !target ? passwordFromSpot(title, null) : null
+    const fromText = verified && !target && !fromTitle ? passwordFromSpot('', xml?.description) : null
+    const status = !verified ? 'unverified' : target ? 'moderation' : 'listed'
+    await db
+      .table('spots')
+      .insert({
+        message_id: messageId,
+        article_number: articleNumber,
+        title: title.slice(0, 2000),
+        tag: header.tag || null,
+        poster: header.poster,
+        category: header.category,
+        subcats: JSON.stringify(header.subcats),
+        size: header.size || null,
+        posted_at: header.stamp ? new Date(header.stamp * 1000) : null,
+        key_id: header.keyId,
+        spotter_id: header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : null,
+        description: xml?.description || null,
+        website: xml?.website || null,
+        nzb_segments: xml?.nzbSegments.length ? xml.nzbSegments : null,
+        password: fromTitle ?? fromText,
+        password_source: fromTitle ? 'spot-title' : fromText ? 'spot-text' : null,
+        status,
+      })
+      .onConflict('message_id')
+      .ignore()
+    if (target) await this.moderate(header.selfSignedPubKey, header.stamp, target)
+    return status
+  }
+
+  /**
+   * A delete/dispose/remove: the target is `removed`, and its links and passwords go from the releases. As in
+   * Spotweb, a personal dispose (the poster's own key) counts only from the same spotter within 5 days; a
+   * moderator's (Spotnet's key) always.
+   */
+  private async moderate(pubKey: string, stamp: number, targetId: string) {
+    const target = await db.from('spots').where('message_id', targetId).first()
+    if (!target) return
+    const personal = pubKey.length > 50 ? spotterId(pubKey) : ''
+    if (personal) {
+      const age = stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
+      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
+    }
+    await db.transaction(async (trx) => {
+      await trx.from('spots').where('message_id', targetId).update({ status: 'removed' })
+      await trx.from('release_spots').where('spot_id', targetId).delete()
+      await trx.from('release_passwords').where('spot_id', targetId).delete()
+      // A release the spot made of its own goes with it.
+      if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', 'spot').delete()
+    })
+    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
+  }
+}

app/validators/user.ts

@@ -14,6 +14,8 @@ export const signupValidator = vine.create({
   email: email().unique({ table: 'users', column: 'email' }),
   password: password(),
   passwordConfirmation: password().sameAs('password'),
+  // Needed when SIGNUP is `invite` (config/access.ts).
+  invite: vine.string().trim().maxLength(32).optional(),
 })
 
 /**

commands/access_invite.ts

@@ -0,0 +1,37 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import AccessService from '#services/access_service'
+import { TIERS } from '#config/access'
+
+/** Makes invite codes (SIGNUP=invite), optionally with a tier for the new account, e.g. a sold VIP year. */
+export default class AccessInvite extends BaseCommand {
+  static commandName = 'access:invite'
+  static description = 'Makes invite codes, optionally giving a tier (e.g. --tier=vip --days=365)'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'How many codes', default: 1 })
+  declare count: number
+
+  @flags.string({ description: `Tier the new account gets (${Object.keys(TIERS).join(', ')})` })
+  declare tier?: string
+
+  @flags.number({ description: 'For how many days the tier holds (without: no end)' })
+  declare days?: number
+
+  @flags.string({ description: 'A note, e.g. who it is for or the payment' })
+  declare note?: string
+
+  async run() {
+    if (this.tier && !TIERS[this.tier]) {
+      this.logger.error(`No such tier: ${this.tier}`)
+      this.exitCode = 1
+      return
+    }
+    const codes = Array.from({ length: this.count }, () => AccessService.newKey())
+    await db.table('invites').multiInsert(
+      codes.map((code) => ({ code, tier: this.tier ?? null, tier_days: this.tier ? (this.days ?? null) : null, note: this.note ?? null }))
+    )
+    for (const code of codes) this.logger.log(code)
+  }
+}

commands/access_suspend.ts

@@ -0,0 +1,28 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import User from '#models/user'
+
+/** Suspends a user (Newznab error 101, the catalog 403), e.g. for a shared key; --undo lifts it. */
+export default class AccessSuspend extends BaseCommand {
+  static commandName = 'access:suspend'
+  static description = 'Suspends a user (or lifts it with --undo)'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'The user’s email' })
+  declare email: string
+
+  @flags.boolean({ description: 'Lift the suspension' })
+  declare undo: boolean
+
+  async run() {
+    const user = await User.findBy('email', this.email)
+    if (!user) {
+      this.logger.error(`No user ${this.email}`)
+      this.exitCode = 1
+      return
+    }
+    user.suspended = !this.undo
+    await user.save()
+    this.logger.info(`${user.email}: ${user.suspended ? 'suspended' : 'active'}`)
+  }
+}

commands/access_tier.ts

@@ -0,0 +1,32 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import User from '#models/user'
+import AccessService from '#services/access_service'
+
+/** Gives a user a tier (a payment, by hand until one is taken on the site); a running one is extended. */
+export default class AccessTier extends BaseCommand {
+  static commandName = 'access:tier'
+  static description = "Sets a user's tier, e.g. access:tier someone@example.com vip --days=365"
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'The user’s email' })
+  declare email: string
+
+  @args.string({ description: 'Tier (config/access.ts)' })
+  declare tier: string
+
+  @flags.number({ description: 'For how many days (without: no end); the same tier still running is extended' })
+  declare days?: number
+
+  async run() {
+    const user = await User.findBy('email', this.email)
+    if (!user) {
+      this.logger.error(`No user ${this.email}`)
+      this.exitCode = 1
+      return
+    }
+    AccessService.grantTier(user, this.tier, this.days ?? null)
+    await user.save()
+    this.logger.info(`${user.email}: ${user.tier} ${user.tierUntil ? `until ${user.tierUntil.toISO()}` : 'without end'}`)
+  }
+}

commands/coordinator_run.ts

@@ -5,13 +5,24 @@ import { Prefetcher } from '#services/prefetcher'
 import Planner from '#services/planner'
 import FragmentMerger from '#services/fragment_merger'
 import Settler from '#services/settler'
+import RangeBlocks from '#services/range_blocks'
 import NntpPool from '#services/nntp_pool'
+import coordinatorConfig from '#config/coordinator'
 
 const TICK_MS = 2000
 const PLAN_EVERY_MS = 60 * 60 * 1000
 const MERGE_EVERY_MS = 60 * 1000
 const MERGE_BUDGET_MS = 30 * 1000
 const STOP_WAIT_MS = 15 * 1000
+/**
+ * Settling in rounds of at most SETTLE_BATCHES batches every few minutes: once an hour, ~3M releases rewrote their
+ * rows and index entries in one go (2.65M in 30 min), the NVMe saturated and the batch stores queued on their locks
+ * (LOG.md 2026-10-16 (67)).
+ */
+const SETTLE_EVERY_MS = 5 * 60 * 1000
+const SETTLE_BATCHES = 4
+/** range_blocks (release_final) after ranges finish. */
+const BLOCKS_EVERY_MS = 5 * 60 * 1000
 
 /**
  * The coordinator's background work (REBUILD.md, part 1): the prefetcher,
@@ -55,17 +66,39 @@ export default class CoordinatorRun extends BaseCommand {
         await sleep(TICK_MS)
       }
     })()
+    // range_blocks in a loop of its own: a settling or merging round doesn't hold release_final back.
+    const blocks = (async () => {
+      while (main && !stopping) {
+        try {
+          await RangeBlocks.rebuild()
+        } catch (error: any) {
+          this.logger.error(`Range blocks failed: ${error.message}`)
+        }
+        for (let waited = 0; waited < BLOCKS_EVERY_MS && !stopping; waited += TICK_MS) await sleep(TICK_MS)
+      }
+    })()
+    // History for new groups hourly; the tail every few minutes.
     let plannedAt = 0
+    let tailedAt = 0
     let mergedAt = 0
+    let settledAt = 0
     while (!stopping) {
       try {
         if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
           plannedAt = Date.now()
           const { groups, ranges } = await Planner.planAll()
           if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
-          const settled = await Settler.run()
+        }
+        if (main && Date.now() - settledAt >= SETTLE_EVERY_MS) {
+          settledAt = Date.now()
+          const settled = await Settler.run(SETTLE_BATCHES)
           if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
         }
+        if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
+          tailedAt = Date.now()
+          const { groups, ranges } = await Planner.planTail()
+          if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
+        }
         if (main && Date.now() - mergedAt >= MERGE_EVERY_MS) {
           mergedAt = Date.now()
           // Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
@@ -90,7 +123,7 @@ export default class CoordinatorRun extends BaseCommand {
       }
       await sleep(TICK_MS)
     }
-    await prefetching
+    await Promise.all([prefetching, blocks])
     // The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
     const waitUntil = Date.now() + STOP_WAIT_MS
     while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)

commands/edge_bench.ts

@@ -78,7 +78,17 @@ export default class EdgeBench extends BaseCommand {
     for (let from = this.start; from <= end; from += this.chunk, index++) {
       const to = Math.min(from + this.chunk - 1, end)
       const started = Date.now()
-      const raw = await client.xoverRaw(from, to)
+      let raw = await client.xoverRaw(from, to)
+      // Eweka cuts some ranges' compressed reply short (the same bytes every time): fetched plain then.
+      try {
+        unzipSync(raw)
+      } catch {
+        const plain = await XoverClient.connect({ host: env.get('NNTP_HOST')!, port: env.get('NNTP_PORT')!, user: env.get('NNTP_USER'), password: env.get('NNTP_PASSWORD'), secure: env.get('NNTP_SECURE', true), compress: false, timeoutMs: 300_000 })
+        await plain.group(this.group)
+        raw = await plain.xoverRaw(from, to)
+        plain.close()
+        this.logger.warning(`chunk ${index}: the compressed reply didn't inflate; fetched plain`)
+      }
       await writeFile(join(this.dir, `${String(index).padStart(4, '0')}-${from}-${to}.gz`), raw)
       this.logger.info(`chunk ${index}: ${from}–${to}, ${(raw.length / 1e6).toFixed(1)} MB in ${((Date.now() - started) / 1000).toFixed(1)} s`)
     }

commands/edge_node.ts

@@ -8,6 +8,8 @@ import { gzip, unzip } from 'node:zlib'
 import { RangeIndexer, decodeFrames, isKeepalive, throttle, toWire, type EdgeBatch, type Lease } from '#lib/edge/index'
 
 const IDLE_MS = 30_000
+/** A batch is posted once this many files (completed, or cut at an edge) wait, whatever its articles. */
+const MAX_PENDING_FILES = 100_000
 const unzipAsync = promisify(unzip)
 const gzipAsync = promisify(gzip)
 const RETRIES = 8
@@ -166,7 +168,8 @@ export default class EdgeNode extends BaseCommand {
       indexer.addText(text.toString('utf8'), chunk.to + 1)
       reachedEnd = chunk.to >= range.end
       if (reachedEnd) indexer.finish()
-      if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles) {
+      // Also early when many files wait: a picture group's 500k articles made batches over the server's 256 MB limit (HTTP 413).
+      if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles || indexer.pending >= MAX_PENDING_FILES) {
         post(indexer.take())
         batchFrom = chunk.to + 1
       }

commands/probe_run.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Prober } from '#services/prober'
+
+/** Phase 2's prober (REBUILD.md, part 6): reads the probe queue's articles and stores what they show. */
+export default class ProbeRun extends BaseCommand {
+  static commandName = 'probe:run'
+  static description = "Reads the probe queue's articles (PROBE_CONNECTIONS) and stores what they show"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  async run() {
+    const prober = new Prober()
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      this.logger.info('Stopping: finishing the articles in flight.')
+      prober.stop()
+      await finished
+    })
+    await prober.run()
+    this.logger.info('Prober stopped.')
+    done!()
+    await this.terminate()
+  }
+}

commands/ranges_plan.ts

@@ -1,18 +1,21 @@
-import { BaseCommand } from '@adonisjs/core/ace'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
 import type { CommandOptions } from '@adonisjs/core/types/ace'
 import Planner from '#services/planner'
 import NntpPool from '#services/nntp_pool'
 
-/** Cuts every active group's history (and new articles since) into ranges. */
+/** Cuts new groups' history into ranges, or (--tail) plans one round of the live tail. */
 export default class RangesPlan extends BaseCommand {
   static commandName = 'ranges:plan'
-  static description = "Plans ranges for the indexed groups' articles"
+  static description = "Plans ranges for new groups' history, or one round of the live tail"
   static options: CommandOptions = { startApp: true }
 
+  @flags.boolean({ description: 'Plan the live tail once (what coordinator:run does every few minutes)' })
+  declare tail: boolean
+
   async run() {
     try {
-      const { groups, ranges } = await Planner.planAll()
+      const { groups, ranges } = this.tail ? await Planner.planTail() : await Planner.planAll()
-      this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
+      this.logger.info(`${ranges} ${this.tail ? 'tail ' : ''}ranges planned over ${groups} groups.`)
     } finally {
       await NntpPool.close()
     }

commands/releases_duplicates.ts

@@ -0,0 +1,82 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+
+/**
+ * How often a release holds a file name more than once (decoded from its
+ * chunks): copies of one upload posted again (multi-posting, reposts that
+ * share a key) or different files that share a name. Copies of the same
+ * size are counted as the same file posted again.
+ */
+export default class ReleasesDuplicates extends BaseCommand {
+  static commandName = 'releases:duplicates'
+  static description = 'Counts file names a release holds more than once, in a sample'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
+  declare sample: number
+
+  async run() {
+    const ids: string[] = (
+      await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])
+    ).rows.map((r: { id: string }) => r.id)
+    const kind = (name: string) =>
+      /\.(jpe?g|png|gif|bmp|webp)$/i.test(name) ? 'image'
+      : /\.(mp3|flac|m4a|ogg|wav)\b/i.test(name) ? 'audio'
+      : /\.(par2)$/i.test(name) ? 'par2'
+      : /\.(rar|r\d\d|7z|zip|\d{3})$/i.test(name) ? 'archive'
+      : /\.(pdf|epub|mobi|cbr|cbz|djvu|chm)$/i.test(name) ? 'document'
+      : /\.(nfo|sfv|nzb|txt|diz)$/i.test(name) ? 'sidecar'
+      : 'other'
+    let releases = 0
+    let withDuplicates = 0
+    let files = 0
+    let extra = 0
+    let completeOnlyByCopies = 0
+    const byKind = new Map<string, number>()
+    const how = { sameFile: 0, differentFile: 0, sameDay: 0, laterRepost: 0 }
+    const examples: string[] = []
+    for (const id of ids) {
+      const release = await db.from('releases').where('id', id).select('name', 'file_total', 'files').first()
+      const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+      const byName = new Map<string, Map<string, { parts: number; bytes: number; at: number | null }>>()
+      for (const c of chunks) {
+        for (const f of decodeChunk(c.payload)) {
+          const copies = byName.get(f.name) ?? new Map()
+          copies.set(f.segments[0]?.[1] ?? '', { parts: f.segments.length, bytes: f.segments.reduce((n, [, , b]) => n + b, 0), at: f.postedAt })
+          byName.set(f.name, copies)
+        }
+      }
+      releases++
+      let dup = false
+      for (const [name, copies] of byName) {
+        files += copies.size
+        if (copies.size < 2) continue
+        dup = true
+        extra += copies.size - 1
+        byKind.set(kind(name), (byKind.get(kind(name)) ?? 0) + copies.size - 1)
+        const list = [...copies.values()]
+        // The same file again: as many parts, within 3% in bytes (article sizes include headers).
+        const same = list.every((c) => c.parts === list[0].parts && Math.abs(c.bytes - list[0].bytes) <= 0.03 * list[0].bytes)
+        if (same) how.sameFile++
+        else how.differentFile++
+        const times = list.map((c) => c.at).filter((t): t is number => t !== null)
+        if (times.length > 1 && Math.max(...times) - Math.min(...times) > 86_400_000) how.laterRepost++
+        else how.sameDay++
+        if (!same && examples.length < 8) examples.push(`${id} ${release.name.slice(0, 60)} | ${name.slice(0, 60)} | ${list.map((c) => `${c.parts}p ${(c.bytes / 1e6).toFixed(1)}MB`).join(', ')}`)
+      }
+      if (!dup) continue
+      withDuplicates++
+      const total = Number(release.file_total ?? 0)
+      if (total && Number(release.files) >= total && byName.size < total) completeOnlyByCopies++
+    }
+    const pct = (n: number, of: number) => `${((100 * n) / Math.max(1, of)).toFixed(2)}%`
+    this.logger.info(`${releases} releases with 2+ files sampled`)
+    this.logger.info(`  with a file name more than once: ${withDuplicates} (${pct(withDuplicates, releases)})`)
+    this.logger.info(`  extra copies: ${extra} of ${files} files (${pct(extra, files)}); by type: ${[...byKind].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${k} ${n}`).join(', ')}`)
+    this.logger.info(`  duplicated names: same file again ${how.sameFile}, different (parts or size) ${how.differentFile}; copies the same day ${how.sameDay}, a day or more apart ${how.laterRepost}`)
+    this.logger.info(`  'complete' only by counting copies: ${completeOnlyByCopies}`)
+    for (const e of examples) this.logger.info(`  different: ${e}`)
+  }
+}

commands/releases_judge.ts

@@ -0,0 +1,38 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { ReleaseJudge, RULES } from '#services/release_judge'
+
+/** The junk judge (REBUILD.md, junk): releases judged after their probes; verdicts hide them from search. */
+export default class ReleasesJudge extends BaseCommand {
+  static commandName = 'releases:judge'
+  static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable), hourly'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  @flags.boolean({ description: 'With --once: count only, write no verdicts' })
+  declare dry: boolean
+
+  async run() {
+    const judge = new ReleaseJudge()
+    if (this.once) {
+      const round = await judge.round(this.dry)
+      for (const rule of RULES) {
+        const r = round.rules[rule.name]
+        if (r) this.logger.info(`${rule.name}: ${r.matched} (+${r.added} −${r.removed}) — ${rule.says}`)
+      }
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      judge.stop()
+      await finished
+    })
+    await judge.run()
+    this.logger.info('Judge stopped.')
+    done!()
+  }
+}

commands/releases_merge.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Par2Merger } from '#services/par2_merger'
+
+/** The par2 merge (REBUILD.md, phase 2): releases whose files one par2 set describes become one. */
+export default class ReleasesMerge extends BaseCommand {
+  static commandName = 'releases:merge'
+  static description = 'Merges the releases whose files one par2 set describes (what the prober read), every minute'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round over every set, then exit' })
+  declare once: boolean
+
+  async run() {
+    const merger = new Par2Merger()
+    if (this.once) {
+      const round = await merger.round()
+      this.logger.info(`${round.sets} sets checked: ${round.releases} releases merged into ${round.merges}.`)
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      merger.stop()
+      await finished
+    })
+    await merger.run()
+    this.logger.info('Merger stopped.')
+    done!()
+    await this.terminate()
+  }
+}

commands/releases_regroup.ts

@@ -0,0 +1,86 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+import { releaseIdentity } from '#lib/release/index'
+
+/**
+ * What lib/release's identity makes of releases already in the catalog: a
+ * sample of releases (by source), their files decoded from the chunks and
+ * each given its release key again. Shows how many would now split, and
+ * into what: for checking an identity change on real data.
+ */
+export default class ReleasesRegroup extends BaseCommand {
+  static commandName = 'releases:regroup'
+  static description = 'Shows how the current release identity would regroup a sample of the catalog'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled per source', default: 2000 })
+  declare sample: number
+
+  @flags.array({ description: 'Release ids to show in full' })
+  declare show?: string[]
+
+  @flags.array({ description: 'Groups whose indexed releases are re-keyed together (merges show too)' })
+  declare group?: string[]
+
+  async run() {
+    for (const name of this.group ?? []) {
+      const { rows } = await db.rawQuery(
+        `SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
+        [name]
+      )
+      const ids = rows.map((r: { release_id: string }) => r.release_id)
+      const all = new Map<string, { name: string; source: string; files: number; subject?: string }>()
+      for (const id of ids) {
+        for (const [key, g] of await this.regroup(Number(id))) {
+          const known = all.get(key)
+          if (known) known.files += g.files
+          else all.set(key, { ...g })
+        }
+      }
+      const sizes = [...all.values()].map((g) => g.files)
+      const single = sizes.filter((n) => n === 1).length
+      this.logger.info(`${name}: ${ids.length} releases now, ${all.size} with the current rule (${single} of one file)`)
+      for (const g of [...all.values()].sort((a, b) => b.files - a.files).slice(0, 6)) this.logger.info(`  ${g.files} files: ${g.name.slice(0, 120)} (${g.source})`)
+      const singles = [...all.values()].filter((g) => g.files === 1)
+      for (let i = 0; i < Math.min(8, singles.length); i++) this.logger.info(`  single: ${singles[Math.floor((i * singles.length) / 8)].subject?.slice(0, 150)}`)
+    }
+    if (this.group?.length) return
+    for (const id of this.show ?? []) {
+      const groups = await this.regroup(Number(id))
+      this.logger.info(`${id}: ${groups.size} release(s) now`)
+      for (const [, g] of groups) this.logger.info(`  ${g.files} files: ${g.name} (${g.source})`)
+    }
+    if (this.show?.length) return
+    for (const source of ['subject', 'post', 'stem']) {
+      const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
+      let split = 0
+      let pieces = 0
+      const examples: string[] = []
+      for (const r of rows) {
+        const groups = await this.regroup(Number(r.id))
+        if (groups.size <= 1) continue
+        split++
+        pieces += groups.size
+        if (examples.length < 5) examples.push(`${r.id} ${r.name} (${r.files} files) → ${groups.size}: ${[...groups.values()].slice(0, 3).map((g) => `${g.name} [${g.files}]`).join(' | ')}`)
+      }
+      this.logger.info(`${source}: ${rows.length} sampled, ${split} would split (into ${pieces} releases)`)
+      for (const e of examples) this.logger.info(`  ${e.slice(0, 260)}`)
+    }
+  }
+
+  private async regroup(id: number) {
+    const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+    const groups = new Map<string, { name: string; source: string; files: number; subject?: string }>()
+    for (const c of chunks) {
+      for (const f of decodeChunk(c.payload)) {
+        const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
+        const g = groups.get(r.key)
+        if (g) g.files++
+        else groups.set(r.key, { name: r.name, source: r.source, files: 1, subject: f.subject })
+      }
+    }
+    return groups
+  }
+}

commands/releases_upload_gaps.ts

@@ -0,0 +1,58 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+
+/**
+ * For telling uploads apart (re-uploads under one release key): how long
+ * one upload takes to post (the span of a release's files' dates, in
+ * releases without a file name twice), and how far apart the copies of a
+ * file name are in releases that have one twice.
+ */
+export default class ReleasesUploadGaps extends BaseCommand {
+  static commandName = 'releases:upload-gaps'
+  static description = 'Posting spans of uploads and gaps between re-uploads, in a sample'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
+  declare sample: number
+
+  async run() {
+    const ids: string[] = (await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])).rows.map((r: { id: string }) => r.id)
+    const bins = [10, 60, 180, 360, 720, 1440, 4320, Number.POSITIVE_INFINITY]
+    const label = (m: number) => (m === Number.POSITIVE_INFINITY ? '>3d' : m >= 1440 ? `≤${m / 1440}d` : m >= 60 ? `≤${m / 60}h` : `≤${m}min`)
+    const spans = new Array(bins.length).fill(0)
+    const gaps = new Array(bins.length).fill(0)
+    const bin = (minutes: number) => bins.findIndex((b) => minutes <= b)
+    for (const id of ids) {
+      const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+      const byName = new Map<string, number[]>()
+      const times: number[] = []
+      for (const c of chunks) {
+        for (const f of decodeChunk(c.payload)) {
+          if (f.postedAt === null) continue
+          times.push(f.postedAt)
+          byName.set(f.name, [...(byName.get(f.name) ?? []), f.postedAt])
+        }
+      }
+      if (times.length < 2) continue
+      const twice = [...byName.values()].filter((t) => t.length > 1)
+      if (twice.length === 0) {
+        spans[bin((Math.max(...times) - Math.min(...times)) / 60_000)]++
+        continue
+      }
+      // The gap between a name's copies (sorted; each step), so a release reposted 3 times counts 2 gaps per name.
+      for (const t of twice) {
+        t.sort((a, b) => a - b)
+        for (let i = 1; i < t.length; i++) gaps[bin((t[i] - t[i - 1]) / 60_000)]++
+      }
+    }
+    const show = (counts: number[]) => {
+      const total = counts.reduce((a, b) => a + b, 0)
+      let cumulative = 0
+      return bins.map((b, i) => `${label(b)} ${((100 * (cumulative += counts[i])) / Math.max(1, total)).toFixed(1)}%`).join(', ') + ` (n=${total})`
+    }
+    this.logger.info(`one upload's posting span (cumulative): ${show(spans)}`)
+    this.logger.info(`gap between copies of a name (cumulative): ${show(gaps)}`)
+  }
+}

commands/rules_signatures.ts

@@ -0,0 +1,52 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { readFile } from 'node:fs/promises'
+import RuleSets from '#services/rule_sets'
+import { signatureKey } from '#lib/rules/index'
+
+/**
+ * Sets the dropped upload signatures (REBUILD.md, rules and recovery) from a
+ * file of "tool<TAB>group" lines, e.g. voyager 1's learned ones:
+ *
+ *   psql … -At -F $'\t' -c "SELECT tool, group_name FROM upload_signatures WHERE dropped"
+ *
+ * Publishes a new rule set version (ranges record the version they ran with).
+ */
+export default class RulesSignatures extends BaseCommand {
+  static commandName = 'rules:signatures'
+  static description = 'Publishes a rule set with the dropped upload signatures from a file'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'File of "tool<TAB>group" lines (more columns and # lines are ignored), e.g. database/data/voyager1_signatures.tsv' })
+  declare file: string
+
+  @flags.string({ description: 'Note stored with the rule set version' })
+  declare note?: string
+
+  @flags.boolean({ description: 'Add to the current signatures instead of replacing them' })
+  declare add: boolean
+
+  async run() {
+    const lines = (await readFile(this.file, 'utf8')).split('\n').map((l) => l.trim()).filter((l) => l && !l.startsWith('#'))
+    const signatures = new Set<string>()
+    for (const line of lines) {
+      const [tool, group] = line.split('\t')
+      if (!tool || !group) throw new Error(`Not "tool<TAB>group": ${line}`)
+      signatures.add(signatureKey(tool, group))
+    }
+    const { version: _v, priorities: _p, ...current } = await RuleSets.current()
+    const before = new Set(current.droppedSignatures)
+    const next = this.add ? new Set([...before, ...signatures]) : signatures
+    if (next.size === before.size && [...next].every((s) => before.has(s))) {
+      this.logger.info(`Unchanged: ${before.size} dropped signatures already.`)
+      return
+    }
+    const version = await RuleSets.publish(
+      { ...current, droppedSignatures: [...next].sort() },
+      this.note ?? `${next.size} dropped signatures from ${this.file}`
+    )
+    const added = [...next].filter((s) => !before.has(s)).length
+    const removed = [...before].filter((s) => !next.has(s)).length
+    this.logger.info(`Rule set ${version}: ${next.size} dropped signatures (${added} added, ${removed} removed).`)
+  }
+}

commands/spots_inspect.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { SpotInspector } from '#services/spot_inspector'
+
+/** The spot inspector (REBUILD.md, spots): each spot's NZB read, its files listed and matched to releases. */
+export default class SpotsInspect extends BaseCommand {
+  static commandName = 'spots:inspect'
+  static description = "Reads each listed spot's NZB, lists its files and links them to the releases they match"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  async run() {
+    const inspector = new SpotInspector()
+    if (this.once) {
+      const round = await inspector.round()
+      this.logger.info(`${round.inspected} spots inspected, ${round.matched} matched releases, ${round.standalone} of their own.`)
+      inspector.stop()
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      inspector.stop()
+      await finished
+    })
+    await inspector.run()
+    this.logger.info('Spot inspector stopped.')
+    done!()
+  }
+}

commands/spots_read.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Spotter } from '#services/spotter'
+
+/** The spotter (REBUILD.md, spots): Spotnet's spots from free.pt's headers into `spots`, on one connection. */
+export default class SpotsRead extends BaseCommand {
+  static commandName = 'spots:read'
+  static description = "Reads Spotnet's spots (free.pt) into the spots table, verified, every few minutes"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  async run() {
+    const spotter = new Spotter()
+    if (this.once) {
+      const round = await spotter.round()
+      this.logger.info(`${round.read} articles read: ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
+      spotter.stop()
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      spotter.stop()
+      await finished
+    })
+    await spotter.run()
+    this.logger.info('Spotter stopped.')
+    done!()
+  }
+}

config/access.ts

@@ -0,0 +1,31 @@
+import env from '#start/env'
+
+/**
+ * Who may use the catalog and the Newznab API.
+ *
+ * `open` (the default): as before, one key for Newznab (NEWZNAB_API_KEY, or
+ * none) and an open web catalog, no limits. `accounts`: every request names a
+ * user (a Newznab `apikey`, or the web interface's token), whose tier sets what
+ * they may do per UTC day and how far back they see.
+ */
+export const ACCESS = env.get('ACCESS', 'open')
+
+/** Who may sign up: anyone, with an unused invite code, or no one. With accounts, invites unless set. */
+export const SIGNUP = env.get('SIGNUP', ACCESS === 'accounts' ? 'invite' : 'open')
+
+export interface TierLimits {
+  /** Searches and release details per day (Newznab t=search & co., the catalog); null: no limit. */
+  apiHits: number | null
+  /** NZBs per day; null: no limit. */
+  grabs: number | null
+  /** How many days back releases are found and fetched; null: everything. */
+  retentionDays: number | null
+}
+
+/** A tier that runs out (users.tier_until) falls back to `free`. */
+export const TIERS: Record<string, TierLimits> = {
+  free: { apiHits: 100, grabs: 10, retentionDays: 365 },
+  vip: { apiHits: 10_000, grabs: 1_000, retentionDays: null },
+  // Staff and friends.
+  unlimited: { apiHits: null, grabs: null, retentionDays: null },
+}

config/coordinator.ts

@@ -12,18 +12,119 @@ export default {
    * 2026-10-09).
    */
   chunkArticles: env.get('CHUNK_ARTICLES', 100_000),
+  /**
+   * The live tail (REBUILD.md, coordinator): new articles become ranges
+   * every `everySeconds` (one LIST ACTIVE per name prefix), leased and
+   * fetched before any history. A group gets a tail range once it has
+   * `minArticles` new (a chunk) or its oldest unplanned article waited
+   * `maxDelayMinutes`, so a quiet group doesn't make a range every round.
+   */
+  tail: {
+    everySeconds: env.get('TAIL_EVERY_SECONDS', 120),
+    minArticles: env.get('TAIL_MIN_ARTICLES', 100_000),
+    maxDelayMinutes: env.get('TAIL_MAX_DELAY_MINUTES', 15),
+    /**
+     * Before everything: the history's 0 (oldest) … 1000 (newest), the stored ranges' -2 and ranges raised to -1 (the
+     * short gaps between done blocks went before the tail at -1 and it fell ~4 h behind; LOG.md 2026-10-16 (70)).
+     */
+    priority: -3,
+  },
   /** A leased range without a heartbeat for this long goes back to the queue. */
   leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
   /** NNTP connections the coordinator fetches headers with. */
   nntpConnections: env.get('NNTP_CONNECTIONS', 4),
-  /** A reply with no data for this long fails (old ranges answer slowly). */
+  /**
-  nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 10 * 60 * 1000),
+   * A reply with no data for this long fails, and its connection is replaced.
+   * Eweka sometimes stops answering on long-lived connections (all 40 quiet,
+   * a fresh one answers in seconds; LOG.md 2026-10-09 (14)): at 10 minutes
+   * the prefetcher nearly stopped. Data arrives within seconds otherwise.
+   */
+  nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 2 * 60 * 1000),
+  /**
+   * A request that got less than this in the last minute (kB) is given up,
+   * its connection closed, and the chunk fetched again: a reply that trickles
+   * (6,204 → 6,229 kB in a minute) never goes quiet long enough for
+   * nntpTimeoutMs, and kept a worker waiting on one chunk for an hour. Not a
+   * deadline: some groups send steadily but slowly (a.b.0day.stuffz ~400 kB a
+   * minute, 5–7 minutes a chunk), and a fixed 90–360 s gave those up again
+   * and again (LOG.md 2026-10-10 (3)).
+   */
+  nntpStallBytesPerMinute: env.get('NNTP_STALL_KB_PER_MINUTE', 100) * 1000,
+  /**
+   * A chunk not fetched after this long is asked again on another
+   * connection; the first answer wins and the other connection is closed.
+   * Eweka stalls single requests: the same chunk took 27–52 s on one
+   * connection and 1–9 s on another at the same moment; median ~3 s
+   * (LOG.md 2026-10-09 (15)). Measured in production (40 + 6 connections vs
+   * 40, 15 min each): 280k vs 273k articles/s, within the noise; the second
+   * request won half the time. Off by default.
+   */
+  nntpHedgeAfterMs: env.get('NNTP_HEDGE_AFTER_SECONDS', 10) * 1000,
+  /** Of nntpConnections, kept for those second requests (the prefetcher's loops are the rest). 0: no hedging. */
+  nntpHedgeConnections: env.get('NNTP_HEDGE_CONNECTIONS', 0),
+  /**
+   * A connection is closed after this long (when it's free), and a fresh one
+   * opened. 0: never. Requests were as often slow (>10 s) on connections
+   * under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)).
+   */
+  nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000,
+  probe: {
+    /**
+     * NNTP connections the prober reads articles with, besides the
+     * coordinator's: the account has 50, the coordinator 40 + 1 plain, and a
+     * couple stay free for downloads. ~7 MB/s each (~10 articles/s).
+     */
+    connections: env.get('PROBE_CONNECTIONS', 6),
+    /** Candidates claimed at a time. */
+    batch: 400,
+    /** BODY commands in flight per connection (pipelined; XoverClient.bodies). */
+    pipeline: env.get('PROBE_PIPELINE', 8),
+  },
+  /**
+   * The catalog's tablespace may grow to this many GB (app/services/catalog_space.ts); above it only the tail and
+   * the stored ranges are leased and prefetched. 0: no cap.
+   */
+  catalogMaxNvmeGb: env.get('CATALOG_MAX_NVME_GB', 0),
+  /**
+   * Spotnet spots (app/services/spotter.ts, spot_inspector.ts): the group the spots are posted in, how many of
+   * its last articles the spotter reads the first time (~215 spots a day: ~3 months), how often it looks again,
+   * the most articles an NZB may take, and how long an inspected spot whose files match no release waits for
+   * the header pass before it becomes a release of its own.
+   */
+  spots: {
+    group: env.get('SPOT_GROUP', 'free.pt'),
+    backfill: env.get('SPOT_BACKFILL', 20_000),
+    everySeconds: env.get('SPOT_EVERY_SECONDS', 120),
+    maxNzbSegments: env.get('SPOT_MAX_NZB_SEGMENTS', 64),
+    standaloneAfterHours: env.get('SPOT_STANDALONE_AFTER_HOURS', 24),
+  },
+  /** Stored XOVER chunks of ranges marked `capture`, for the grouping sweeps (app/services/corpus.ts). Off when unset. */
+  corpus: {
+    dir: env.get('CORPUS_DIR'),
+    /**
+     * Capture ranges fetched into the corpus by connections the spool leaves idle, this many at a time (8, 2 connections each, left 20 of 40 idle): the
+     * spool only fetches what workers lease soon, and the tail comes first, so after a reset the capture
+     * ranges waited behind it for hours (1,400 marked, none stored, 34 of 40 connections idle; LOG.md 2026-10-12 (27)).
+     */
+    ahead: env.get('CORPUS_AHEAD', 16),
+  },
   spool: {
     dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
     /** Ranges kept fetched ahead, per active lease. */
-    aheadPerLease: env.get('SPOOL_AHEAD_PER_LEASE', 2),
+    aheadPerLease: env.get('SPOOL_AHEAD_PER_LEASE', 4),
     /** At least this many ranges ahead, so the first leases start at once. */
     minAhead: env.get('SPOOL_MIN_AHEAD', 2),
     maxBytes: env.get('SPOOL_MAX_GB', 10) * 1e9,
+    /**
+     * At most this many connections on one range, and on one group, at a
+     * time (0: no limit): spreads the requests over ranges and groups instead
+     * of all on the next chunks of the first ranges. Homelab, 40 connections,
+     * 12 min each, alternated: unlimited (on ~10 ranges) 281k and 167k
+     * articles/s, 2 per range and 4 per group (on ~22) 319k and 296k, with
+     * shorter chunk times (11.5 vs 13–24 s; LOG.md 2026-10-09 (18)). Needs
+     * targets for it: aheadPerLease 4.
+     */
+    perRange: env.get('PREFETCH_PER_RANGE', 2),
+    perGroup: env.get('PREFETCH_PER_GROUP', 4),
   },
 }

config/static.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from '@adonisjs/static'
+
+/**
+ * Configuration options to tweak the static files middleware.
+ * The complete set of options are documented on the
+ * official documentation website.
+ *
+ * https://docs.adonisjs.com/guides/static-assets
+ */
+const staticServerConfig = defineConfig({
+  enabled: true,
+  etag: true,
+  lastModified: true,
+  dotFiles: 'ignore',
+})
+
+export default staticServerConfig

database/data/voyager1_signatures.tsv

@@ -0,0 +1,30 @@
+# Voyager 1 upload_signatures WHERE dropped, 2026-10-09: tool, group, probed, named
+other	alt.binaries.ath	10988	24
+ngPost	alt.binaries.ath	3843	0
+hex32	alt.binaries.test	1890	2
+ngPost	alt.binaries.a51	1516	0
+nyuu	alt.binaries.a51	700	0
+nyuu	alt.binaries.mom	656	0
+hex32	alt.binaries.boneless	453	1
+nyuu	alt.binaries.misc	356	0
+nyuu	alt.binaries.games.encrypted	353	0
+other	alt.binaries.etc	317	1
+other	alt.binaries.misc	310	0
+nyuu	alt.binaries.warez	292	0
+other	alt.binaries.usenet2day	255	0
+other	alt.binaries.ijsklontje	238	0
+ngPost	alt.binaries.bloaf	237	0
+nyuu	alt.binaries.nl	196	0
+other	alt.binaries.archive.encrypted	189	0
+nyuu	alt.binaries.frogs	172	0
+other	alt.binaries.encrypted	160	0
+yuun	alt.binaries.nl	152	0
+ngPost	alt.binaries.nl	149	0
+other	alt.binaries.bloaf	134	0
+nyuu	alt.binaries.erotica	119	0
+other	alt.binaries.warez	119	0
+other	alt.binaries.ftn	107	0
+nyuu	alt.binaries.ijsklontje	106	0
+random	alt.binaries.newznzb.delta	103	0
+ngPost	alt.binaries.mom	102	0
+random	alt.binaries.newznzb.charlie	101	0

database/migrations/1770000001200_release_category.ts

@@ -0,0 +1,39 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A release's Newznab category from its name (REBUILD.md, part 9): voyager
+ * 1's release_category() without the archive contents (phase 2 has them).
+ * TV (5030/5040/5045), movies (2030/2040/2045), audio (3010/3040),
+ * books (7020), otherwise 8010.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(String.raw`
+      CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
+        SELECT CASE
+          -- TV: S01E02, 1x02, or a season pack (S01).
+          WHEN name ~* '(^|[._ -])(S\d{1,2}E\d{1,3}|\d{1,2}x\d{2})([._ -]|$)' OR name ~* '[._ -]S\d{1,2}([._ -]|$)' THEN
+            CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
+                 WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
+          -- Movies: a year plus a source or quality marker.
+          WHEN name ~* '(^|[._ (-])(19|20)\d{2}([._ )-]|$)'
+               AND name ~* '(480p|576p|720p|1080p|2160p|blu-?ray|web-?dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.?26[45]|hevc|remux)' THEN
+            CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
+                 WHEN name ~* '720p|1080p|blu-?ray' THEN 2040 ELSE 2030 END
+          -- Audio: lossless, mp3, or scene music names (-WEB-2026-GROUP).
+          WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
+          WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)\d{2}-[A-Za-z0-9]+$' THEN 3010
+          WHEN name ~* '(^|[._ -])(epub|ebook|mobi)([._ -]|$)' THEN 7020
+          ELSE 8010
+        END
+      $$`)
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS category integer GENERATED ALWAYS AS (release_category(name)) STORED`)
+    // Newznab and the web interface list a category newest first.
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS releases_category_posted_idx ON releases (category, posted_at DESC)`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS category`)
+    this.schema.raw(`DROP FUNCTION IF EXISTS release_category(text)`)
+  }
+}

database/migrations/1770000001300_groups_tail.ts

@@ -0,0 +1,15 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A group's live tail can be paused (its history and its place in the owner
+ * rule stay): e.g. to let other groups' history go first.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE groups ADD COLUMN IF NOT EXISTS tail boolean NOT NULL DEFAULT true`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE groups DROP COLUMN IF EXISTS tail`)
+  }
+}

database/migrations/1770000001400_release_uploads.ts

@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Re-uploads as releases of their own (REBUILD.md, decision d): releases
+ * keep the identity key their files gave (identity_key; release_key is that
+ * for the first upload, then key:2, key:3, …), the group that owns their
+ * articles, and their posting window (posted_at … last_posted_at). A batch's
+ * chunk joins a release with its key and group whose window is within
+ * UPLOAD_GAP_MS, else starts one.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS identity_key varchar(40)`)
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS group_id integer`)
+    this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS last_posted_at timestamptz`)
+    this.schema.raw(`UPDATE releases SET identity_key = release_key, last_posted_at = posted_at WHERE identity_key IS NULL`)
+    this.schema.raw(`ALTER TABLE releases ALTER COLUMN identity_key SET NOT NULL`)
+    this.schema.raw(`ALTER TABLE releases ALTER COLUMN release_key TYPE varchar(48)`)
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS releases_identity_idx ON releases (identity_key, group_id)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX IF EXISTS releases_identity_idx`)
+    this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS identity_key, DROP COLUMN IF EXISTS group_id, DROP COLUMN IF EXISTS last_posted_at`)
+  }
+}

database/migrations/1770000001500_ranges_plain_chunks.ts

@@ -0,0 +1,18 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Chunks of a range to fetch without compression: Eweka sends some ranges'
+ * compressed XOVER truncated, the same bytes every time, while the plain
+ * reply is whole (LOG.md 2026-10-09 (11)). A worker reports such a chunk;
+ * its start goes here and the prefetcher fetches it plain.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS plain_chunks bigint[] NOT NULL DEFAULT '{}'`)
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS error text`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS plain_chunks, DROP COLUMN IF EXISTS error`)
+  }
+}

database/migrations/1770000001600_tsm_system_rows.ts

@@ -0,0 +1,12 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** TABLESAMPLE SYSTEM_ROWS: the stats' sample of the catalog at a fixed size (a percentage read 30 s of random pages on the HDD at 7M releases). */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`CREATE EXTENSION IF NOT EXISTS tsm_system_rows`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP EXTENSION IF EXISTS tsm_system_rows`)
+  }
+}

database/migrations/1770000001700_release_inspections.ts

@@ -0,0 +1,38 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * Phase 2's probe results (REBUILD.md, part 6): per release and probe kind
+ * what its article showed (yEnc name, content type, a few names inside,
+ * encryption), in CHUNK_TABLESPACE (the HDD) when set: they're read per
+ * release, not searched. On the catalog only what search needs: whether the
+ * release is passworded and its best inner name.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    const where = tablespace ? ` TABLESPACE ${tablespace}` : ''
+    this.schema.raw(`
+      CREATE TABLE release_inspections (
+        release_id bigint NOT NULL,
+        kind varchar(8) NOT NULL,
+        result jsonb NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (release_id, kind)${tablespace ? ` USING INDEX TABLESPACE ${tablespace}` : ''}
+      )${where}`)
+    this.schema.alterTable('releases', (table) => {
+      table.boolean('passworded').nullable()
+      table.string('inner_name', 255).nullable()
+      table.timestamp('inspected_at', { useTz: true }).nullable()
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => {
+      table.dropColumn('passworded')
+      table.dropColumn('inner_name')
+      table.dropColumn('inspected_at')
+    })
+    this.schema.dropTable('release_inspections')
+  }
+}

database/migrations/1770000001800_release_inspections_per_candidate.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** One inspection per probe candidate: a release can have several of a kind (two RAR sets). */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`DELETE FROM release_inspections`)
+    this.schema.raw(`ALTER TABLE release_inspections DROP CONSTRAINT release_inspections_pkey`)
+    this.schema.raw(`ALTER TABLE release_inspections ADD COLUMN candidate_id bigint NOT NULL`)
+    this.schema.raw(`ALTER TABLE release_inspections ADD PRIMARY KEY (release_id, candidate_id)`)
+    // Their results went with the deleted rows: probed again.
+    this.schema.raw(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'done'`)
+    this.schema.raw(`UPDATE releases SET passworded = NULL, inner_name = NULL, inspected_at = NULL WHERE inspected_at IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE release_inspections DROP CONSTRAINT release_inspections_pkey`)
+    this.schema.raw(`ALTER TABLE release_inspections DROP COLUMN candidate_id`)
+    this.schema.raw(`ALTER TABLE release_inspections ADD PRIMARY KEY (release_id, kind)`)
+  }
+}

database/migrations/1770000001900_releases_inner_name_trgm.ts

@@ -0,0 +1,18 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Search matches the prober's inner names too (SearchService): a trigram
+ * index on them, only where there is one. Built concurrently: the catalog
+ * keeps taking batches meanwhile.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_inner_name_trgm ON releases USING gin (inner_name gin_trgm_ops) WHERE inner_name IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_inner_name_trgm`)
+  }
+}

database/migrations/1770000002000_releases_sort_indexes.ts

@@ -0,0 +1,25 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Indexes that match the newest-first sort (SearchService: posted_at DESC
+ * NULLS LAST, id DESC). The old ones (posted_at DESC: nulls first) didn't,
+ * so the default page sorted all 68M releases: 23 s. Built concurrently, the
+ * old ones dropped after.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`)
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_category_newest_idx ON releases (category, posted_at DESC NULLS LAST, id DESC)`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_posted_at_idx`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_category_posted_idx`)
+  }
+
+  async down() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_posted_at_idx ON releases (posted_at DESC)`)
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_category_posted_idx ON releases (category, posted_at DESC)`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_newest_idx`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_category_newest_idx`)
+  }
+}

database/migrations/1770000002100_ranges_capture.ts

@@ -0,0 +1,12 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** Ranges whose XOVER chunks are kept in the corpus (CORPUS_DIR) for the grouping sweeps. */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS capture boolean NOT NULL DEFAULT false`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS capture`)
+  }
+}

database/migrations/1770000002200_par2_merge.ts

@@ -0,0 +1,60 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The par2 merge (REBUILD.md, phase 2; app/services/par2_merger.ts): what the
+ * prober read that ties files to a par2 set, and the merges made from it.
+ * - file_hashes: per probed article that starts a file, the MD5 of its first 16 KiB and the file's size.
+ * - par2_files: per par2 recovery set, every file it describes (FileDesc: the same 16 KiB MD5, length, name).
+ * - par2_sets: which releases hold a par2 file of a set.
+ * - release_merges: releases merged into another, by set.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE file_hashes (
+        candidate_id bigint PRIMARY KEY,
+        release_id bigint NOT NULL,
+        hash16k char(32) NOT NULL,
+        size bigint NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+    this.schema.raw(`CREATE INDEX file_hashes_hash_idx ON file_hashes (hash16k)`)
+    this.schema.raw(`CREATE INDEX file_hashes_release_idx ON file_hashes (release_id)`)
+    this.schema.raw(`CREATE INDEX file_hashes_created_idx ON file_hashes (created_at)`)
+    this.schema.raw(`
+      CREATE TABLE par2_files (
+        set_id char(32) NOT NULL,
+        file_id char(32) NOT NULL,
+        hash16k char(32) NOT NULL,
+        length bigint NOT NULL,
+        name text NOT NULL,
+        PRIMARY KEY (set_id, file_id)
+      )`)
+    this.schema.raw(`CREATE INDEX par2_files_hash_idx ON par2_files (hash16k)`)
+    this.schema.raw(`
+      CREATE TABLE par2_sets (
+        set_id char(32) NOT NULL,
+        release_id bigint NOT NULL,
+        files integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (set_id, release_id)
+      )`)
+    this.schema.raw(`CREATE INDEX par2_sets_release_idx ON par2_sets (release_id)`)
+    this.schema.raw(`CREATE INDEX par2_sets_created_idx ON par2_sets (created_at)`)
+    this.schema.raw(`
+      CREATE TABLE release_merges (
+        id bigserial PRIMARY KEY,
+        from_id bigint NOT NULL,
+        from_key varchar(48) NOT NULL,
+        into_id bigint NOT NULL,
+        set_id char(32) NOT NULL,
+        files integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+    this.schema.raw(`CREATE INDEX release_merges_into_idx ON release_merges (into_id)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS file_hashes, par2_files, par2_sets, release_merges`)
+  }
+}

database/migrations/1770000002300_probe_cache.ts

@@ -0,0 +1,25 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * What a probed article showed, by message-id (app/services/prober.ts): an
+ * article's inspection doesn't depend on the grouping rules, so a catalog
+ * reset (which empties the probe queue) needn't read it from the news server
+ * again. `version` is PROBE_VERSION when it was inspected: a change to the
+ * inspection makes older rows stale. Not truncated by a reset.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE probe_cache (
+        message_id text PRIMARY KEY,
+        status varchar(8) NOT NULL,
+        result jsonb,
+        version integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS probe_cache`)
+  }
+}

database/migrations/1770000002400_probe_candidates_kind_index.ts

@@ -0,0 +1,19 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The prober claims waiting candidates by kind from random points of the
+ * queue (Prober.claim): by the id index alone a slice filtered ~700k rows for
+ * the rarer kinds (0.37 s a slice, 20 slices and up to 5 tiers a claim, the
+ * lanes waiting meanwhile; LOG.md 2026-10-12 (17)). Built concurrently.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_kind_waiting_idx ON probe_candidates (kind, id) WHERE status = 'waiting'`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_kind_waiting_idx`)
+  }
+}

database/migrations/1770000002500_probe_candidates_ready.ts

@@ -0,0 +1,30 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Candidates the prober may claim (Prober.claim): `ready` once their release
+ * settled (the settler sets it), `unreadable` when that release had no
+ * readable name then. The claims joined releases for both and started at
+ * random points of the queue; in its newer half nothing had settled, so a
+ * slice scanned to the end (0.3 s a slice, 20 slices, 5 tiers: the prober
+ * stood still for minutes per claim; LOG.md 2026-10-12 (20)). With an index
+ * over the claimable rows only.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`ALTER TABLE probe_candidates ADD COLUMN IF NOT EXISTS ready boolean NOT NULL DEFAULT false`)
+    this.schema.raw(`ALTER TABLE probe_candidates ADD COLUMN IF NOT EXISTS unreadable boolean NOT NULL DEFAULT false`)
+    this.schema.raw(
+      `UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
+       WHERE r.id = pc.release_id AND r.settled_at IS NOT NULL AND NOT pc.ready`
+    )
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_ready_idx ON probe_candidates (kind, unreadable, id) WHERE status = 'waiting' AND ready`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_kind_waiting_idx`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_ready_idx`)
+    this.schema.raw(`ALTER TABLE probe_candidates DROP COLUMN IF EXISTS ready, DROP COLUMN IF EXISTS unreadable`)
+  }
+}

database/migrations/1770000002600_inner_stem_merge.ts

@@ -0,0 +1,30 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The merge by inner name (app/services/par2_merger.ts, InnerStemMerger):
+ * obfuscated uploads post each file under a subject name of its own, but the
+ * yEnc name inside its first article shares a random stem with the rest of
+ * the upload ("[1/7] - "471044d8…"" holds "bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001";
+ * hep, hephaestus, gyges: 31,000 one-file releases; LOG.md 2026-10-12 (32)).
+ * - An index over a lone inspection's stem: the yEnc name up to its first dot, when 16–64 letters and digits.
+ * - release_merges: which kind of merge (par2 or inner), and room for a stem in set_id.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS release_inspections_inner_stem_idx ON release_inspections
+       ((substring(result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.'))) WHERE kind = 'lone'`
+    )
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS release_inspections_created_idx ON release_inspections (created_at)`)
+    this.schema.raw(`ALTER TABLE release_merges ALTER COLUMN set_id TYPE varchar(64)`)
+    this.schema.raw(`ALTER TABLE release_merges ADD COLUMN IF NOT EXISTS via varchar(8) NOT NULL DEFAULT 'par2'`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS release_inspections_inner_stem_idx`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS release_inspections_created_idx`)
+    this.schema.raw(`ALTER TABLE release_merges DROP COLUMN IF EXISTS via`)
+  }
+}

database/migrations/1770000002700_release_inspections_catalog_tablespace.ts

@@ -0,0 +1,27 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * release_inspections in the catalog's tablespace (default_tablespace: nvme
+ * on homelab), not CHUNK_TABLESPACE: the prober writes it every flush and
+ * the merger looks stems up in it (release_inspections_inner_stem_idx), so it
+ * isn't only read per release any more (migration 1700). Its flushes took
+ * 35–83 s a minute with the table on the HDD (LOG.md 2026-10-13 (43)).
+ * Homelab was moved by hand first; here a no-op when it's there already, or
+ * without a default tablespace (tests).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const { rows } = await db.rawQuery(
+        `SELECT current_setting('default_tablespace') AS target,
+                (SELECT t.spcname FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.relname = 'release_inspections') AS current`
+      )
+      const { target, current } = rows[0]
+      if (!target || current === target) return
+      await db.rawQuery(`ALTER TABLE release_inspections SET TABLESPACE ${target}`)
+      await db.rawQuery(`ALTER INDEX release_inspections_pkey SET TABLESPACE ${target}`)
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000002800_spots.ts

@@ -0,0 +1,96 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Spotnet spots (REBUILD.md, spots; after voyager 1's D29), in two processes:
+ * - the spotter (`spots:read`) reads free.pt's headers, verifies each spot's
+ *   signature from its full header (X-XML) and keeps it in `spots`, with a
+ *   password its title or description names;
+ * - the spot inspector (`spots:inspect`) reads each spot's compressed NZB
+ *   (alt.binaries.ftd), keeps its files in `spot_files`, and matches them to
+ *   the releases the header pass built (the same release identity), linking
+ *   them in `release_spots`. A spot whose files match no release becomes a
+ *   release of its own (source `spot`) a day after its inspection (until
+ *   the header pass builds its releases after all: migration 2900).
+ * The NZB itself is kept (gzip) for such a release's downloads.
+ * Passwords are kept apart from the catalog in `release_passwords`, one row
+ * per release and password, with where it came from (the NZB's meta, the
+ * spot's title or its text); the catalog only says `passworded`.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE spots (
+        message_id text PRIMARY KEY,
+        article_number bigint,
+        title text NOT NULL,
+        tag text,
+        poster text NOT NULL,
+        category integer,
+        subcats jsonb,
+        size bigint,
+        posted_at timestamptz,
+        key_id integer,
+        spotter_id text,
+        description text,
+        website text,
+        nzb_segments text[],
+        -- The NZB as read (gzip): a release the spot makes of its own has no chunks to build one from.
+        nzb bytea,
+        password text,
+        password_source varchar(16),
+        -- listed: verified, waiting for the inspector; inspected, no-nzb, dummy, nzb-gone, nzb-unreadable: after it;
+        -- moderation: a delete/dispose message; removed: disposed of by a moderator or its spotter.
+        status varchar(16) NOT NULL DEFAULT 'listed',
+        files integer,
+        matched integer,
+        release_id bigint,
+        error text,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        inspected_at timestamptz
+      )`)
+    this.schema.raw(`CREATE INDEX spots_listed_idx ON spots (posted_at DESC) WHERE status = 'listed'`)
+    this.schema.raw(`CREATE INDEX spots_unmatched_idx ON spots (inspected_at) WHERE status = 'inspected' AND matched = 0 AND release_id IS NULL`)
+    this.schema.raw(`
+      CREATE TABLE spot_files (
+        spot_id text NOT NULL,
+        idx integer NOT NULL,
+        name text NOT NULL,
+        subject text NOT NULL,
+        poster text,
+        groups text[],
+        posted_at timestamptz,
+        bytes bigint NOT NULL,
+        segments integer NOT NULL,
+        release_id bigint,
+        PRIMARY KEY (spot_id, idx)
+      )`)
+    this.schema.raw(`CREATE INDEX spot_files_release_idx ON spot_files (release_id) WHERE release_id IS NOT NULL`)
+    this.schema.raw(`
+      CREATE TABLE release_spots (
+        release_id bigint NOT NULL,
+        spot_id text NOT NULL,
+        files integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (release_id, spot_id)
+      )`)
+    this.schema.raw(`CREATE INDEX release_spots_spot_idx ON release_spots (spot_id)`)
+    this.schema.raw(`
+      CREATE TABLE release_passwords (
+        release_id bigint NOT NULL,
+        password text NOT NULL,
+        source varchar(16) NOT NULL,
+        spot_id text,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (release_id, password)
+      )`)
+    this.schema.raw(`
+      CREATE TABLE spot_state (
+        group_name text PRIMARY KEY,
+        last_article bigint NOT NULL
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS spots, spot_files, release_spots, release_passwords, spot_state`)
+  }
+}

database/migrations/1770000002900_spots_rematched.ts

@@ -0,0 +1,21 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Unmatched spots are matched again in turns (`rematched_at`), for 30 days after their inspection, also once they
+ * became a release of their own: the header pass may build their releases later (a spot's history before the
+ * catalog's), and then the spot moves to that release (LOG.md 2026-10-13 (45)).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS rematched_at timestamptz`)
+    // Inspected before the inspector set it (2026-10-13).
+    this.schema.raw(`UPDATE spots SET matched = 0 WHERE status = 'inspected' AND matched IS NULL`)
+    this.schema.raw(`DROP INDEX IF EXISTS spots_unmatched_idx`)
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS spots_unmatched_idx ON spots (coalesce(rematched_at, inspected_at)) WHERE status = 'inspected' AND matched = 0`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX IF EXISTS spots_unmatched_idx`)
+    this.schema.raw(`ALTER TABLE spots DROP COLUMN IF EXISTS rematched_at`)
+  }
+}

database/migrations/1770000003000_spots_nzb_only_unmatched.ts

@@ -0,0 +1,10 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** A matched spot's NZB isn't kept (its releases have their chunks): only a spot of its own needs one (2.7 GB at 9k spots). */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`UPDATE spots SET nzb = NULL WHERE matched > 0 AND nzb IS NOT NULL`)
+  }
+
+  async down() {}
+}

database/migrations/1770000003100_chunk_indexes_with_chunks.ts

@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * release_chunks' indexes with its chunks in CHUNK_TABLESPACE (the HDD), not the catalog's tablespace: a chunk is
+ * read from the HDD anyway, and the October partition's three indexes took 2.9 GB of the NVMe (its pkey never
+ * scanned; LOG.md 2026-10-14 (54)). The parent's partitioned indexes too, so new partitions put theirs there.
+ * Homelab was moved by hand first; a no-op where they're there already.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    if (!tablespace) return
+    this.defer(async (db) => {
+      const { rows } = await db.rawQuery(
+        `SELECT i.relname FROM pg_class c JOIN pg_index x ON x.indrelid = c.oid JOIN pg_class i ON i.oid = x.indexrelid
+         LEFT JOIN pg_tablespace t ON t.oid = i.reltablespace
+         WHERE c.relname LIKE 'release_chunks%' AND coalesce(t.spcname, 'pg_default') <> ?`,
+        [tablespace]
+      )
+      for (const { relname } of rows) await db.rawQuery(`ALTER INDEX ${relname} SET TABLESPACE ${tablespace}`)
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003200_rarely_read_tables_with_chunks.ts

@@ -0,0 +1,32 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * par2_files, fragments and probe_cache, with their indexes, in CHUNK_TABLESPACE (the HDD): ~20 GB of the NVMe at
+ * 42M releases, read by key only (the merger, the fragment merger, the prober). The catalog was at its
+ * CATALOG_MAX_NVME_GB cap and history paused (LOG.md 2026-10-14 (55)). Homelab was moved by hand first (3 min,
+ * with the writers stopped); a no-op where they're there already.
+ */
+const TABLES = ['par2_files', 'fragments', 'probe_cache']
+
+export default class extends BaseSchema {
+  async up() {
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    if (!tablespace) return
+    this.defer(async (db) => {
+      for (const table of TABLES) {
+        const { rows } = await db.rawQuery(
+          `SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+           WHERE (c.relname = ? OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = ?::regclass))
+             AND coalesce(t.spcname, 'pg_default') <> ?`,
+          [table, table, tablespace]
+        )
+        for (const { relname, relkind } of rows) {
+          await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${tablespace}`)
+        }
+      }
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003300_access.ts

@@ -0,0 +1,49 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Accounts with limits (config/access.ts): a Newznab key and a tier per user,
+ * invites, and what each user used per day.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('users', (table) => {
+      // Existing users get a key of their own (a volatile default is evaluated per row).
+      table.string('api_key', 32).notNullable().unique().defaultTo(this.raw(`md5(random()::text || clock_timestamp()::text)`))
+      table.string('tier', 16).notNullable().defaultTo('free')
+      // Null: the tier doesn't run out.
+      table.timestamp('tier_until', { useTz: true }).nullable()
+      table.boolean('suspended').notNullable().defaultTo(false)
+    })
+
+    this.schema.createTable('invites', (table) => {
+      table.string('code', 32).primary()
+      // What the invite gives (e.g. a sold VIP invite); null: the free tier.
+      table.string('tier', 16).nullable()
+      table.integer('tier_days').nullable()
+      table.string('note').nullable()
+      table.integer('used_by').nullable().references('id').inTable('users').onDelete('SET NULL')
+      table.timestamp('used_at', { useTz: true }).nullable()
+      table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
+    })
+
+    this.schema.createTable('usage', (table) => {
+      table.integer('user_id').notNullable().references('id').inTable('users').onDelete('CASCADE')
+      // A UTC day.
+      table.date('day').notNullable()
+      table.integer('api_hits').notNullable().defaultTo(0)
+      table.integer('grabs').notNullable().defaultTo(0)
+      table.primary(['user_id', 'day'])
+    })
+  }
+
+  async down() {
+    this.schema.dropTable('usage')
+    this.schema.dropTable('invites')
+    this.schema.alterTable('users', (table) => {
+      table.dropColumn('api_key')
+      table.dropColumn('tier')
+      table.dropColumn('tier_until')
+      table.dropColumn('suspended')
+    })
+  }
+}

database/migrations/1770000003300_release_verdicts.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The junk judge's verdicts (app/services/release_judge.ts): per release the rule that judged it, after its
+ * probes, and the rule set's version. Search leaves judged releases out. Hidden only for now (nothing dropped):
+ * a verdict whose rule no longer matches (a password found, a spot linked) is taken back by the next round.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE release_verdicts (
+        release_id bigint PRIMARY KEY,
+        rule varchar(16) NOT NULL,
+        verdict varchar(8) NOT NULL DEFAULT 'hidden',
+        rule_version integer NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+    this.schema.raw(`CREATE INDEX release_verdicts_rule_idx ON release_verdicts (rule)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS release_verdicts`)
+  }
+}

database/migrations/1770000003400_fold_candidates.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Small releases the settler just settled (at most 3 files, under 1 MB), queued for the merger's sidecar fold
+ * (app/services/sidecar_folder.ts): an .nfo, .sfv or .nzb that came apart from its upload (another counter, another
+ * name) joins it. A queue, so the fold needn't scan the releases (LOG.md 2026-10-15 (57)).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE fold_candidates (
+        release_id bigint PRIMARY KEY,
+        created_at timestamptz NOT NULL DEFAULT now()
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS fold_candidates`)
+  }
+}

database/migrations/1770000003500_fragments_back_on_nvme.ts

@@ -0,0 +1,29 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * fragments, with its indexes, back in the catalog's tablespace (default_tablespace: nvme on homelab), out of
+ * CHUNK_TABLESPACE where 3200 put it: it isn't read by key only. The fragment merger walks its file_key index and
+ * reads each row, written and deleted all the time; on the HDD one round of its 30 s budget took 12+ minutes, and the
+ * coordinator's settling and tail planning waited behind it (LOG.md 2026-10-15 (57)). 13 GB. Homelab was moved by
+ * hand first, with the writers stopped; a no-op where it's there already, or without a default tablespace (tests).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
+      const target = setting[0].target
+      if (!target) return
+      const { rows } = await db.rawQuery(
+        `SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+         WHERE (c.relname = 'fragments' OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = 'fragments'::regclass))
+           AND coalesce(t.spcname, 'pg_default') <> ?`,
+        [target]
+      )
+      for (const { relname, relkind } of rows) {
+        await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`)
+      }
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003600_worker_cursors.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Where a worker's incremental rounds go on after a restart. The merger kept its "since" in memory only: each deploy
+ * or pause started it from 1970, a first round over every par2 set and inner stem ever found (25+ min with par2_files
+ * on the HDD) before the sidecar fold's turn (LOG.md 2026-10-15 (58)).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE worker_cursors (
+        name text PRIMARY KEY,
+        at timestamptz NOT NULL
+      )`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP TABLE IF EXISTS worker_cursors`)
+  }
+}

database/migrations/1770000003700_par2_files_back_on_nvme.ts

@@ -0,0 +1,29 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * par2_files, with its indexes, back in the catalog's tablespace (default_tablespace: nvme on homelab), out of
+ * CHUNK_TABLESPACE where 3200 put it: the merger matches every new file hash against it and reads each set's rows, a
+ * random read per row on the HDD (~1,090 sets/min, a full round ~28 h; LOG.md 2026-10-15 (58)). Homelab rebuilt it
+ * there by hand without the complete and judged sets' rows (app/services/par2_pruner.ts; LOG.md 2026-10-16 (60)); a
+ * no-op where it's there already, or without a default tablespace (tests).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
+      const target = setting[0].target
+      if (!target) return
+      const { rows } = await db.rawQuery(
+        `SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+         WHERE (c.relname = 'par2_files' OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = 'par2_files'::regclass))
+           AND coalesce(t.spcname, 'pg_default') <> ?`,
+        [target]
+      )
+      for (const { relname, relkind } of rows) {
+        await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`)
+      }
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003800_rules_dropped_posters.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A rule set version with droppedPosters (lib/rules/rule_set.ts): the newest set plus the CelebMovie family, whose
+ * scraped celebrity pictures workers drop at the header pass from now on (LOG.md 2026-10-16 (65)). A new version, so the
+ * ranges record which ran with it. Nothing where there's no rule set yet (the default gets it) or the newest has it.
+ */
+const POSTERS = ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes']
+
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const latest = await db.from('rule_sets').orderBy('version', 'desc').first()
+      if (!latest || 'droppedPosters' in latest.rules) return
+      await db.table('rule_sets').insert({
+        version: Number(latest.version) + 1,
+        rules: JSON.stringify({ ...latest.rules, droppedPosters: POSTERS }),
+        note: 'dropped posters: the CelebMovie family (2026-10-16)',
+      })
+    })
+  }
+
+  async down() {}
+}

database/migrations/1770000003900_final_completeness.ts

@@ -0,0 +1,45 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Final completeness by time window (LOG.md 2026-10-16 (67)). A release settles once quiet, but history is read
+ * newest first and in islands: one next to an unread range was "incomplete" twice as often (52.5% against 25.7%),
+ * its missing files only not read yet. An upload is posted within hours, so a release is final (its complete verdict
+ * definite) once done ranges cover its posting times and FINAL_MARGIN either side.
+ * - ranges.posted_low/high: the posting time span the workers report per batch (postedSpan: without the earliest and
+ *   latest 1%), the min and max over the range's batches.
+ * - range_blocks: runs of consecutive done ranges per group with their span (RangeBlocks rebuilds it); at_group_start
+ *   when the run starts at the group's first range (nothing older to wait for).
+ * - release_final(): whether one block holds the release's articles and reaches FINAL_MARGIN past its posting times.
+ *   Computed when asked, so no release row is written (50M updates would double the table on the NVMe).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS posted_low timestamptz, ADD COLUMN IF NOT EXISTS posted_high timestamptz`)
+    this.schema.raw(`
+      CREATE TABLE range_blocks (
+        group_id integer NOT NULL,
+        start_id bigint NOT NULL,
+        end_id bigint NOT NULL,
+        posted_low timestamptz,
+        posted_high timestamptz,
+        at_group_start boolean NOT NULL,
+        PRIMARY KEY (group_id, start_id)
+      )`)
+    // FINAL_MARGIN: 6 hours.
+    this.schema.raw(`
+      CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz)
+      RETURNS boolean LANGUAGE sql STABLE AS $$
+        SELECT EXISTS (
+          SELECT 1 FROM range_blocks b
+          WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article
+            AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours')
+            AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours')
+      $$`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`)
+    this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
+    this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS posted_low, DROP COLUMN IF EXISTS posted_high`)
+  }
+}

database/migrations/1770000004000_posters.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Poster search (LOG.md 2026-10-16 (70)): "poster ILIKE '%…%'" on releases had no index, so it walked all 50M releases
+ * newest first; a poster with few releases took 30+ minutes. posters holds each poster once (3M, many random per file
+ * of an obfuscated upload) with a trigram index; releases_poster_newest_idx gives a poster's releases newest first.
+ * Both by the poster's first 200 characters: a spam poster of 3,024 bytes is over a btree entry's 2,704. The batch store
+ * adds new posters. Homelab built the releases index by hand first (concurrently); a no-op there.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE TABLE IF NOT EXISTS posters (poster text PRIMARY KEY)`)
+    this.schema.raw(`INSERT INTO posters (poster) SELECT DISTINCT left(poster, 200) FROM releases WHERE poster IS NOT NULL ON CONFLICT DO NOTHING`)
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_poster_newest_idx ON releases (left(poster, 200), posted_at DESC NULLS LAST, id DESC)`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_poster_newest_idx`)
+    this.schema.raw(`DROP TABLE IF EXISTS posters`)
+  }
+}

database/migrations/1770000004100_release_final_complete.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * release_final() with the release's complete flag: a release with every file its counter announced is final as it is
+ * (nothing it lacks can come); the time window only decides for the incomplete and those without a counter (LOG.md
+ * 2026-10-16 (70): a complete episode posted an hour before was "pending" for six hours more).
+ */
+export default class extends BaseSchema {
+  async up() {
+    // Beside the five-argument one, which the services still running call until the deploy restarts them.
+    this.schema.raw(`
+      CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz, complete boolean)
+      RETURNS boolean LANGUAGE sql STABLE AS $$
+        SELECT coalesce(complete, false) OR EXISTS (
+          SELECT 1 FROM range_blocks b
+          WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article
+            AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours')
+            AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours')
+      $$`)
+  }
+
+  async down() {}
+}

database/schema.ts

@@ -32,6 +32,21 @@ export class AuthAccessTokenSchema extends BaseModel {
   declare updatedAt: DateTime | null
 }
 
+export class FileHashSchema extends BaseModel {
+  static $columns = ['candidateId', 'createdAt', 'hash16K', 'releaseId', 'size'] as const
+  $columns = FileHashSchema.$columns
+  @column({ isPrimary: true })
+  declare candidateId: bigint | number
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare hash16K: string
+  @column()
+  declare releaseId: bigint | number
+  @column()
+  declare size: bigint | number
+}
+
 export class FragmentSchema extends BaseModel {
   static $columns = ['articleNumber', 'fileKey', 'fragment', 'id', 'rangeId', 'segments'] as const
   $columns = FragmentSchema.$columns
@@ -50,7 +65,7 @@ export class FragmentSchema extends BaseModel {
 }
 
 export class GroupSchema extends BaseModel {
-  static $columns = ['active', 'createdAt', 'high', 'id', 'low', 'name', 'plannedAt', 'priority'] as const
+  static $columns = ['active', 'createdAt', 'high', 'id', 'low', 'name', 'plannedAt', 'priority', 'tail'] as const
   $columns = GroupSchema.$columns
   @column()
   declare active: boolean
@@ -68,6 +83,27 @@ export class GroupSchema extends BaseModel {
   declare plannedAt: DateTime | null
   @column()
   declare priority: number
+  @column()
+  declare tail: boolean
+}
+
+export class InviteSchema extends BaseModel {
+  static $columns = ['code', 'createdAt', 'note', 'tier', 'tierDays', 'usedAt', 'usedBy'] as const
+  $columns = InviteSchema.$columns
+  @column({ isPrimary: true })
+  declare code: string
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare note: string | null
+  @column()
+  declare tier: string | null
+  @column()
+  declare tierDays: number | null
+  @column.dateTime()
+  declare usedAt: DateTime | null
+  @column()
+  declare usedBy: number | null
 }
 
 export class NodeAccessTokenSchema extends BaseModel {
@@ -108,8 +144,51 @@ export class NodeSchema extends BaseModel {
   declare name: string
 }
 
+export class Par2FileSchema extends BaseModel {
+  static $columns = ['fileId', 'hash16K', 'length', 'name', 'setId'] as const
+  $columns = Par2FileSchema.$columns
+  @column()
+  declare fileId: string
+  @column()
+  declare hash16K: string
+  @column()
+  declare length: bigint | number
+  @column()
+  declare name: string
+  @column({ isPrimary: true })
+  declare setId: string
+}
+
+export class Par2SetSchema extends BaseModel {
+  static $columns = ['createdAt', 'files', 'releaseId', 'setId'] as const
+  $columns = Par2SetSchema.$columns
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare files: number
+  @column()
+  declare releaseId: bigint | number
+  @column({ isPrimary: true })
+  declare setId: string
+}
+
+export class ProbeCacheSchema extends BaseModel {
+  static $columns = ['createdAt', 'messageId', 'result', 'status', 'version'] as const
+  $columns = ProbeCacheSchema.$columns
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column({ isPrimary: true })
+  declare messageId: string
+  @column()
+  declare result: any | null
+  @column()
+  declare status: string
+  @column()
+  declare version: number
+}
+
 export class ProbeCandidateSchema extends BaseModel {
-  static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'releaseId', 'status'] as const
+  static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'ready', 'releaseId', 'status', 'unreadable'] as const
   $columns = ProbeCandidateSchema.$columns
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
@@ -122,16 +201,22 @@ export class ProbeCandidateSchema extends BaseModel {
   @column()
   declare messageId: string
   @column()
+  declare ready: boolean
+  @column()
   declare releaseId: bigint | number
   @column()
   declare status: string
+  @column()
+  declare unreadable: boolean
 }
 
 export class RangeSchema extends BaseModel {
-  static $columns = ['attempts', 'doneAt', 'endId', 'error', 'groupId', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'ruleVersion', 'spooledAt', 'spooledChunks', 'startId', 'status'] as const
+  static $columns = ['attempts', 'capture', 'doneAt', 'endId', 'error', 'groupId', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'plainChunks', 'priority', 'ruleVersion', 'spooledAt', 'spooledChunks', 'startId', 'status'] as const
   $columns = RangeSchema.$columns
   @column()
   declare attempts: number
+  @column()
+  declare capture: boolean
   @column.dateTime()
   declare doneAt: DateTime | null
   @column()
@@ -153,6 +238,8 @@ export class RangeSchema extends BaseModel {
   @column()
   declare nodeId: number | null
   @column()
+  declare plainChunks: any
+  @column()
   declare priority: number
   @column()
   declare ruleVersion: number | null
@@ -187,12 +274,48 @@ export class ReleaseChunkSchema extends BaseModel {
   declare segments: number
 }
 
+export class ReleaseInspectionSchema extends BaseModel {
+  static $columns = ['candidateId', 'createdAt', 'kind', 'releaseId', 'result'] as const
+  $columns = ReleaseInspectionSchema.$columns
+  @column()
+  declare candidateId: bigint | number
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare kind: string
+  @column({ isPrimary: true })
+  declare releaseId: bigint | number
+  @column()
+  declare result: any
+}
+
+export class ReleaseMergeSchema extends BaseModel {
+  static $columns = ['createdAt', 'files', 'fromId', 'fromKey', 'id', 'intoId', 'setId'] as const
+  $columns = ReleaseMergeSchema.$columns
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+  @column()
+  declare files: number
+  @column()
+  declare fromId: bigint | number
+  @column()
+  declare fromKey: string
+  @column({ isPrimary: true })
+  declare id: bigint | number
+  @column()
+  declare intoId: bigint | number
+  @column()
+  declare setId: string
+}
+
 export class ReleaseSchema extends BaseModel {
-  static $columns = ['bytes', 'chunks', 'complete', 'fileTotal', 'files', 'firstArticle', 'firstSeenAt', 'groups', 'hasPar2', 'id', 'lastArticle', 'lastSeenAt', 'name', 'postedAt', 'poster', 'readable', 'releaseKey', 'segments', 'settledAt', 'source'] as const
+  static $columns = ['bytes', 'category', 'chunks', 'complete', 'fileTotal', 'files', 'firstArticle', 'firstSeenAt', 'groupId', 'groups', 'hasPar2', 'id', 'identityKey', 'innerName', 'inspectedAt', 'lastArticle', 'lastPostedAt', 'lastSeenAt', 'name', 'passworded', 'postedAt', 'poster', 'readable', 'releaseKey', 'segments', 'settledAt', 'source'] as const
   $columns = ReleaseSchema.$columns
   @column()
   declare bytes: bigint | number
   @column()
+  declare category: number | null
+  @column()
   declare chunks: number
   @column()
   declare complete: boolean | null
@@ -205,17 +328,29 @@ export class ReleaseSchema extends BaseModel {
   @column.dateTime()
   declare firstSeenAt: DateTime
   @column()
+  declare groupId: number | null
+  @column()
   declare groups: any
   @column()
   declare hasPar2: boolean
   @column({ isPrimary: true })
   declare id: bigint | number
   @column()
+  declare identityKey: string
+  @column()
+  declare innerName: string | null
+  @column.dateTime()
+  declare inspectedAt: DateTime | null
+  @column()
   declare lastArticle: bigint | number | null
   @column.dateTime()
+  declare lastPostedAt: DateTime | null
+  @column.dateTime()
   declare lastSeenAt: DateTime
   @column()
   declare name: string
+  @column()
+  declare passworded: boolean | null
   @column.dateTime()
   declare postedAt: DateTime | null
   @column()
@@ -283,9 +418,24 @@ export class RuleSetSchema extends BaseModel {
   declare version: number
 }
 
+export class UsageSchema extends BaseModel {
+  static $columns = ['apiHits', 'day', 'grabs', 'userId'] as const
+  $columns = UsageSchema.$columns
+  @column()
+  declare apiHits: number
+  @column.date()
+  declare day: DateTime
+  @column()
+  declare grabs: number
+  @column({ isPrimary: true })
+  declare userId: number
+}
+
 export class UserSchema extends BaseModel {
-  static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
+  static $columns = ['apiKey', 'createdAt', 'email', 'fullName', 'id', 'password', 'suspended', 'tier', 'tierUntil', 'updatedAt'] as const
   $columns = UserSchema.$columns
+  @column()
+  declare apiKey: string
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
   @column()
@@ -296,6 +446,12 @@ export class UserSchema extends BaseModel {
   declare id: number
   @column({ serializeAs: null })
   declare password: string
+  @column()
+  declare suspended: boolean
+  @column()
+  declare tier: string
+  @column.dateTime()
+  declare tierUntil: DateTime | null
   @column.dateTime({ autoCreate: true, autoUpdate: true })
   declare updatedAt: DateTime | null
 }

deploy/README.md

@@ -0,0 +1,76 @@
+# Deploying voyager 2
+
+Voyager 2 (branch `rebuild`) runs on homelab as the compose project
+`usenet-rebuild`, separate from voyager 1's (`usenet-indexer`, branch
+`master`): its own Postgres, network, images and directory.
+
+A push to `rebuild` that passes CI is deployed: the deploy job SSHes to
+homelab with the deploy key, whose forced command
+(`~/bin/deploy-usenet-indexer`) passes `deploy-rebuild <sha>` on to
+`~/bin/deploy-usenet-rebuild` (this directory's `deploy.sh`).
+
+## What runs
+
+| Service | Image | Does |
+|---|---|---|
+| `postgres` | postgres:17 | the catalog (tablespace `nvme`) and the WAL (`PG_WAL_DIR`, `data/pg_wal` a symlink to it) on the NVMe; the chunks (HDD) |
+| `migrate` | `usenet-rebuild` (target `app`) | `migration:run`, before every deploy |
+| `web` | `usenet-rebuild` | catalog API, Newznab, web interface; port 3334 (`HTTP_ROUTES=public`) |
+| `node-api` | `usenet-rebuild` | the edge workers' API (leases, streams, batches); port 3335 on the compose network only (`HTTP_ROUTES=node`) |
+| `coordinator` | `usenet-rebuild` | prefetching (all NNTP connections), planning, the tail, merging, settling |
+| `prober` | `usenet-rebuild` | phase 2: reads the probe queue's articles over `PROBE_CONNECTIONS` (6) connections, stores what they show |
+| `merger` | `usenet-rebuild` | the par2 merge: releases whose files one par2 set describes become one (no NNTP) |
+| `judge` | `usenet-rebuild` | the junk judge: releases judged hourly by what their probes showed (dead, fake, repeated, locked, unusable); verdicts in `release_verdicts` hide them from search (no NNTP) |
+| `spotter` | `usenet-rebuild` | Spotnet: free.pt's spots into `spots`, signatures checked, the passwords they name kept; 1 connection |
+| `spot-inspector` | `usenet-rebuild` | each spot's NZB: its files in `spot_files`, linked to the releases they match (`release_spots`), passwords in `release_passwords`; 1 connection |
+| `edge` | `usenet-rebuild-edge` (target `edge`) | the Rust worker, `--threads $EDGE_THREADS` (8) |
+| `edge-ts` | `usenet-rebuild` | the TypeScript worker, profile `edge-ts`, not started |
+| `ace` | `usenet-rebuild` | one-off ace commands, profile `tools` |
+
+## On homelab
+
+`/mnt/bulk/usenet-rebuild` (`/home` is nearly full):
+
+| Path | |
+|---|---|
+| `compose.yaml` | copied from `src/deploy/compose.yaml` on every deploy |
+| `.env` | credentials and tuning, mode 600; not in git (below) |
+| `src/` | the code at the deployed commit (the build context) |
+| `pg/` | Postgres data (HDD) |
+| `spool/` | prefetched XOVER chunks |
+| `corpus/` | stored XOVER chunks of ranges marked `capture`, for the grouping sweeps (up to 300 GB) |
+| `corpus-out/` | the sweeps' replays, per edge image (`python3 src/deploy/corpus_sweep.py`) |
+| `deployed.log` | one line per deploy: sha, time (local) |
+| `.cache/repo.git` | bare clone the deploys fetch into |
+| `app/` | an older copy of the code with dev dependencies, for `tools/*.ts` (`r.sh`); not what runs |
+
+The catalog's tablespace is `/var/lib/usenet-rebuild-catalog` on the NVMe
+(`PG_NVME_DIR`). Keep 10 GB free on the root filesystem.
+
+`.env` holds `APP_KEY`, `DB_USER`, `DB_PASSWORD`, `DB_DATABASE`, the `NNTP_*`
+account, `NODE_TOKEN` (the edge worker's token, from `nodes:create`),
+`CHUNK_TABLESPACE=pg_default`, `TZ`, `LOG_LEVEL`, `SESSION_DRIVER` and
+optional tuning (`NNTP_CONNECTIONS`, `EDGE_THREADS`, `PG_SHARED_BUFFERS`,
+anything in `start/env.ts`). Hosts, ports and paths are set in compose.yaml.
+
+## By hand
+
+```sh
+cd /mnt/bulk/usenet-rebuild
+~/bin/deploy-usenet-rebuild <sha>          # what CI runs
+docker compose ps
+docker compose logs -f --since 5m coordinator
+docker compose run --rm ace releases:duplicates
+docker compose stop edge coordinator       # pause indexing (the worker first)
+docker compose up -d --no-deps edge coordinator
+```
+
+- Recreate app services only, with `--no-deps`; Postgres only on purpose,
+  with the worker and coordinator stopped first.
+- Stop `edge` before `node-api`: a worker holding a stream keeps the server
+  from exiting.
+- The NNTP account is shared with voyager 1 and downloads: `NNTP_CONNECTIONS`
+  (40) plus one plain connection, plus the prober's `PROBE_CONNECTIONS` (6),
+  out of 50.
+- `deploy.sh` doesn't update itself: after changing it, install it again
+  (`install -m 755 src/deploy/deploy.sh ~/bin/deploy-usenet-rebuild`).

deploy/compose.yaml

@@ -0,0 +1,197 @@
+# Voyager 2 on homelab (deploy/README.md). deploy.sh copies this file to the
+# deployment root (/mnt/bulk/usenet-rebuild), next to src/ (the code at the
+# deployed commit), .env (credentials, tuning), pg/ and spool/; run
+# `docker compose` there. Nothing is shared with voyager 1's compose project.
+name: usenet-rebuild
+
+x-app: &app
+  image: usenet-rebuild:latest
+  build:
+    context: ./src
+    target: app
+  restart: unless-stopped
+  env_file: .env
+  environment: &app-env
+    NODE_ENV: production
+    HOST: 0.0.0.0
+    PORT: 3334
+    DB_HOST: postgres
+    DB_PORT: 5432
+    APP_URL: http://web:3334
+    SPOOL_DIR: /spool
+    # Stored XOVER chunks of the ranges marked `capture` (the grouping sweeps' corpus, HDD; app/services/corpus.ts).
+    CORPUS_DIR: /corpus
+  volumes:
+    - ./spool:/spool
+    - ./corpus:/corpus
+  stop_grace_period: 60s
+  depends_on:
+    migrate:
+      condition: service_completed_successfully
+
+services:
+  postgres:
+    image: postgres:17
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_DB: ${DB_DATABASE}
+    volumes:
+      # Chunks (write-once payloads) on the HDD; the catalog in tablespace
+      # nvme, the database's default_tablespace (LOG.md 2026-10-09 (14)).
+      - ./pg:/var/lib/postgresql/data
+      - ${PG_NVME_DIR:-/var/lib/usenet-rebuild-catalog}:/var/lib/postgresql/nvme
+      # The WAL on the NVMe too: data/pg_wal is a symlink to /var/lib/postgresql/wal. On the HDD every commit
+      # waited on it (WALWrite, 56–70 ms a write with the HDD 100% busy; LOG.md 2026-10-13 (43)).
+      - ${PG_WAL_DIR:-/var/lib/usenet-rebuild-wal}:/var/lib/postgresql/wal
+    # synchronous_commit = off is set with ALTER SYSTEM (postgresql.auto.conf in the data dir).
+    # Memory for 62 GiB of RAM shared with other services (LOG.md 2026-10-16 (69)): the catalog's hot part in
+    # shared_buffers, the planner told the OS caches the rest; random_page_cost for the NVMe (the HDD tablespace
+    # pg_default keeps 4: ALTER TABLESPACE … SET (random_page_cost = 4)).
+    command:
+      - postgres
+      - -c
+      - shared_preload_libraries=pg_stat_statements
+      - -c
+      - shared_buffers=${PG_SHARED_BUFFERS:-8GB}
+      - -c
+      - effective_cache_size=${PG_EFFECTIVE_CACHE_SIZE:-32GB}
+      - -c
+      - random_page_cost=1.1
+      - -c
+      - work_mem=16MB
+      - -c
+      - maintenance_work_mem=1GB
+      - -c
+      - max_wal_size=8GB
+      - -c
+      - checkpoint_timeout=15min
+      - -c
+      - wal_buffers=64MB
+    # Parallel hash joins' shared memory: 1g failed with "could not resize shared memory segment" (LOG.md 2026-10-16 (61)).
+    shm_size: 2g
+    stop_grace_period: 120s
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
+      interval: 5s
+      retries: 30
+
+  migrate:
+    <<: *app
+    restart: 'no'
+    command: ['node', 'ace', 'migration:run', '--force']
+    volumes: []
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  # Catalog API, Newznab and the web interface.
+  web:
+    <<: *app
+    command: ['node', 'bin/server.js']
+    environment:
+      <<: *app-env
+      HTTP_ROUTES: public
+    ports:
+      - '3334:3334'
+
+  # The edge workers' API (leases, streams, batches), apart from the web
+  # interface: streams hold a connection up to an hour, batches of up to
+  # 256 MB are parsed in-process. Only the compose network reaches it.
+  node-api:
+    <<: *app
+    command: ['node', 'bin/server.js']
+    environment:
+      <<: *app-env
+      HTTP_ROUTES: node
+      PORT: 3335
+
+  # Prefetching (all NNTP connections), planning, the tail, merging, settling.
+  coordinator:
+    <<: *app
+    command: ['node', 'ace', 'coordinator:run']
+    environment:
+      <<: *app-env
+      NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-38}
+
+  # Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS
+  # connections of its own. The account has 50: the coordinator's 38 (+ its
+  # plain one), the prober's 8, the spotter's and the spot inspector's 1 each,
+  # 2 to spare (a hedged request closes its connection and opens another)
+  # (with 10 the spot inspector got 502 Too many connections).
+  prober:
+    <<: *app
+    command: ['node', 'ace', 'probe:run']
+    volumes: []
+    environment:
+      <<: *app-env
+      PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-8}
+
+  # The par2 merge: releases whose files one par2 set describes (what the prober read) become one. No NNTP.
+  merger:
+    <<: *app
+    command: ['node', 'ace', 'releases:merge']
+    volumes: []
+
+  # The junk judge: releases judged hourly by what their probes showed (dead, fake, repeated, locked, unusable);
+  # verdicts hide them from search. No NNTP.
+  judge:
+    <<: *app
+    command: ['node', 'ace', 'releases:judge']
+    volumes: []
+
+  # Spotnet: the spotter reads free.pt's spots (verified, with the passwords
+  # they name) into the spots table; the spot inspector reads each spot's NZB,
+  # lists its files and links the releases they match. One connection each.
+  spotter:
+    <<: *app
+    command: ['node', 'ace', 'spots:read']
+    volumes: []
+
+  spot-inspector:
+    <<: *app
+    command: ['node', 'ace', 'spots:inspect']
+    volumes: []
+
+  # The Rust worker; it has no NNTP account and talks to node-api only.
+  # Stopped before node-api (a worker holding a stream keeps the server from
+  # exiting).
+  edge:
+    image: usenet-rebuild-edge:latest
+    build:
+      context: ./src
+      target: edge
+    restart: unless-stopped
+    command: ['--threads', '${EDGE_THREADS:-8}']
+    environment:
+      COORDINATOR_URL: http://node-api:3335
+      NODE_TOKEN: ${NODE_TOKEN}
+    stop_grace_period: 60s
+    depends_on:
+      node-api:
+        condition: service_started
+
+  # The TypeScript worker, for comparing: docker compose --profile edge-ts up -d edge-ts
+  edge-ts:
+    <<: *app
+    profiles: [edge-ts]
+    command: ['node', 'ace', 'edge:node', '--processes', '${EDGE_THREADS:-8}']
+    environment:
+      <<: *app-env
+      COORDINATOR_URL: http://node-api:3335
+    depends_on:
+      node-api:
+        condition: service_started
+
+  # One-off ace commands: docker compose run --rm ace releases:duplicates
+  ace:
+    <<: *app
+    profiles: [tools]
+    restart: 'no'
+    entrypoint: ['node', 'ace']
+    command: ['list']
+
+networks:
+  default:
+    name: usenet-rebuild

deploy/corpus_sweep.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+"""The grouping sweeps' corpus (REBUILD.md, corpus): stored XOVER chunks replayed with each build's rules.
+
+Run on homelab from the deployment root (/mnt/bulk/usenet-rebuild):
+
+  python3 src/deploy/corpus_sweep.py capture [N]   mark N more ranges to store (default 100), fetched first
+  python3 src/deploy/corpus_sweep.py replay        edge-bench over every stored range with the deployed edge image
+  python3 src/deploy/corpus_sweep.py report        what the replay grouped: suspects, and the change since the last build
+
+The prefetcher stores the chunks of ranges marked `capture` in corpus/<group>/<start>-<end>/ (range.json once
+complete) and reads them from there afterwards, so a reset doesn't fetch them again. Replays go to
+corpus-out/<image id>/<group>/<start>-<end>.jsonl, one per build; `report` compares the newest two.
+"""
+import collections
+import json
+import os
+import shutil
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+ROOT = os.getcwd()
+CORPUS = os.path.join(ROOT, 'corpus')
+OUT = os.path.join(ROOT, 'corpus-out')
+IMAGE = 'usenet-rebuild-edge:latest'
+# The corpus may take this much of the HDD (user, 2026-10-12: up to 300 GB).
+MAX_BYTES = 300e9
+# Ranks in groups.tsv the new ranges come from first (the top 100), then the rest of what's indexed.
+TOP = 100
+
+
+def env(name):
+    for line in open(os.path.join(ROOT, '.env')):
+        if line.startswith(name + '='):
+            return line.split('=', 1)[1].strip()
+    raise SystemExit(f'{name} not in .env')
+
+
+def psql(sql):
+    """Runs SQL in the catalog; rows as lists of strings."""
+    out = subprocess.run(
+        ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', env('DB_USER'), '-d', env('DB_DATABASE'), '-At', '-F', '\t', '-v', 'ON_ERROR_STOP=1'],
+        input=sql, capture_output=True, text=True, check=True,
+    ).stdout
+    # Without the command tag a RETURNING prints after its rows ("UPDATE 100").
+    return [line.split('\t') for line in out.splitlines() if line and not line.startswith(('UPDATE ', 'INSERT ', 'DELETE '))]
+
+
+def corpus_bytes():
+    total = 0
+    for dirpath, _, files in os.walk(CORPUS):
+        total += sum(os.path.getsize(os.path.join(dirpath, f)) for f in files)
+    return total
+
+
+def capture(n):
+    used = corpus_bytes()
+    # Marked ranges not complete yet take room too: at the stored ranges' mean size, less what they hold already
+    # (2026-10-13: 2,300 complete at 296.6 GB and 100 more marked would have made ~310 GB).
+    complete = sum(1 for _ in complete_ranges())
+    marked = int(psql('SELECT count(*) FROM ranges WHERE capture;')[0][0])
+    mean = used / complete if complete else 0
+    pending = max(0, marked - complete) * mean
+    room = MAX_BYTES - max(used, complete * mean + pending)
+    n = min(n, int(room // mean)) if mean else n
+    if n <= 0:
+        print(f'corpus at {used / 1e9:.1f} GB of {MAX_BYTES / 1e9:.0f} (with {marked - complete} ranges still filling, ~{(complete * mean + pending) / 1e9:.0f} GB): nothing more marked')
+        return
+    top = [line.split('\t')[1] for line in open(os.path.join(ROOT, 'groups.tsv')).read().splitlines()[:TOP]]
+    names = ','.join("'" + g.replace("'", "''") + "'" for g in top)
+    # Spread over the groups and their history: per group, the ranges farthest from those stored already
+    # (at random in a group without any), the top-100 groups first. Only ranges not started (a range half
+    # spooled would miss its first chunks; a done one is stored at the next reset, which re-reads them all).
+    # Priority 0: fetched before the rest of the history (the tail stays first).
+    rows = psql(f"""
+WITH stored AS (SELECT group_id, array_agg(start_id) AS starts FROM ranges WHERE capture GROUP BY 1),
+pick AS (
+  SELECT r.id, g.name IN ({names}) AS top, row_number() OVER (PARTITION BY r.group_id ORDER BY
+    coalesce((SELECT min(abs(r.start_id - s)) FROM unnest(st.starts) s), 0) DESC, random()) AS n
+  FROM ranges r JOIN groups g ON g.id = r.group_id LEFT JOIN stored st ON st.group_id = r.group_id
+  WHERE r.priority >= 0 AND NOT r.capture AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.next_id = r.start_id
+),
+chosen AS (SELECT id FROM pick ORDER BY top DESC, n, random() LIMIT {int(n)})
+UPDATE ranges r SET capture = true, priority = 0 FROM chosen WHERE r.id = chosen.id RETURNING r.id;
+""")
+    total = psql('SELECT count(*), count(*) FILTER (WHERE status = \'done\') FROM ranges WHERE capture;')[0]
+    print(f'marked {len(rows)} ranges; {total[0]} marked in all, {total[1]} indexed; corpus {used / 1e9:.1f} GB')
+
+
+def complete_ranges():
+    for group in sorted(os.listdir(CORPUS)) if os.path.isdir(CORPUS) else []:
+        for rng in sorted(os.listdir(os.path.join(CORPUS, group))):
+            if os.path.exists(os.path.join(CORPUS, group, rng, 'range.json')):
+                yield group, rng
+
+
+def image_id():
+    return subprocess.run(['docker', 'image', 'inspect', '-f', '{{.Id}}', IMAGE], capture_output=True, text=True, check=True).stdout.strip().split(':')[-1][:12]
+
+
+def replay():
+    build = image_id()
+    out = os.path.join(OUT, build)
+    todo = [(g, r) for g, r in complete_ranges() if not os.path.exists(os.path.join(out, g, r + '.jsonl'))]
+    for g, _ in todo:
+        os.makedirs(os.path.join(out, g), exist_ok=True)
+    print(f'build {build}: {len(todo)} ranges to replay', flush=True)
+    if not todo:
+        return
+    # One container, 6 at a time (the edge worker keeps its threads).
+    script = 'xargs -P 6 -n 2 sh -c \'edge-bench "/corpus/$0/$1" --releases "/out/$0/$1.jsonl.part" >/dev/null 2>&1 && mv "/out/$0/$1.jsonl.part" "/out/$0/$1.jsonl"\''
+    started = datetime.now()
+    subprocess.run(
+        ['docker', 'run', '--rm', '-i', '--user', '1000:1000', '-v', f'{CORPUS}:/corpus:ro', '-v', f'{out}:/out', '--entrypoint', 'sh', IMAGE, '-c', script],
+        input=''.join(f'{g} {r}\n' for g, r in todo), text=True, check=True,
+    )
+    print(f'replayed in {(datetime.now() - started).total_seconds():.0f} s')
+    # Older builds' replays beyond the last two go.
+    builds = sorted((os.path.getmtime(os.path.join(OUT, b)), b) for b in os.listdir(OUT))
+    for _, b in builds[:-2]:
+        shutil.rmtree(os.path.join(OUT, b))
+
+
+def replayed(build):
+    """The ranges a build replayed, as paths relative to its directory."""
+    root = os.path.join(OUT, build)
+    return {os.path.relpath(os.path.join(d, f), root) for d, _, files in os.walk(root) for f in files if f.endswith('.jsonl')}
+
+
+def load(build, only=None):
+    """Releases of one build's replay (of the ranges in `only`, when given), merged by (group, key) as the catalog does."""
+    releases = {}
+    for rel in sorted(replayed(build)):
+        if only is not None and rel not in only:
+            continue
+        dirpath, f = os.path.split(os.path.join(OUT, build, rel))
+        if True:
+            for line in open(os.path.join(dirpath, f), encoding='utf-8', errors='replace'):
+                r = json.loads(line)
+                k = (r['group'], r['key'])
+                have = releases.get(k)
+                if have is None:
+                    releases[k] = r
+                else:
+                    have['files'] += r['files']
+                    have['segments'] += r['segments']
+                    have['subjects'] = (have['subjects'] + r['subjects'])[:3]
+                    have['fileTotal'] = have['fileTotal'] or r['fileTotal']
+    return releases
+
+
+def hour(r):
+    return datetime.fromtimestamp((r['postedAt'] or 0) / 1000, timezone.utc).strftime('%Y-%m-%d %H')
+
+
+def show(r, prefix='    '):
+    print(f"{prefix}{r['files']} files, total {r['fileTotal']}, {r['source']}: {r['name'][:80]!r} ({r['poster'][:30]}, {r['group']})")
+    for s in r['subjects'][:2]:
+        print(f'{prefix}  {s[:150]}')
+
+
+def report():
+    builds = sorted((os.path.getmtime(os.path.join(OUT, b)), b) for b in os.listdir(OUT)) if os.path.isdir(OUT) else []
+    if not builds:
+        raise SystemExit('nothing replayed yet')
+    build = builds[-1][1]
+    rel = load(build)
+    ranges = sum(1 for _ in complete_ranges())
+    print(f'build {build}: {len(rel):,} releases, {sum(r["files"] for r in rel.values()):,} files from {ranges} stored ranges ({corpus_bytes() / 1e9:.1f} GB)')
+
+    print('\n== same name, poster and hour: 4+ releases (under-grouping)')
+    clusters = collections.defaultdict(list)
+    for r in rel.values():
+        clusters[(r['group'], r['poster'], r['name'], hour(r))].append(r)
+    for (g, p, name, h), rs in sorted(clusters.items(), key=lambda kv: -len(kv[1]))[:25]:
+        if len(rs) < 4:
+            break
+        print(f"  {len(rs)} releases, {sum(r['files'] for r in rs)} files: {name[:70]!r} ({p[:25]}, {g}, {h})")
+        for s in rs[0]['subjects'][:1] + rs[-1]['subjects'][:1]:
+            print(f'      {s[:150]}')
+
+    print('\n== single files of a counter of 5+, per group and poster (under-grouping)')
+    singles = collections.defaultdict(list)
+    for r in rel.values():
+        if r['files'] == 1 and (r['fileTotal'] or 0) >= 5:
+            singles[(r['group'], r['poster'])].append(r)
+    for (g, p), rs in sorted(singles.items(), key=lambda kv: -len(kv[1]))[:15]:
+        if len(rs) < 20:
+            break
+        print(f'  {len(rs)} single files: {p[:30]} ({g})')
+        for s in rs[0]['subjects'][:1] + rs[-1]['subjects'][:1]:
+            print(f'      {s[:150]}')
+
+    print('\n== 2000+ files, or 3x the counter (over-grouping)')
+    big = [r for r in rel.values() if r['files'] > 2000 or (r['fileTotal'] and r['files'] > 3 * r['fileTotal'] and r['files'] > 50)]
+    for r in sorted(big, key=lambda r: -r['files'])[:20]:
+        show(r, '  ')
+
+    if len(builds) >= 2:
+        # Over the ranges both replayed: the corpus grows between builds.
+        both = replayed(builds[-2][1]) & replayed(build)
+        before, after = load(builds[-2][1], both), load(build, both)
+        print(f'\n== change since build {builds[-2][1]} on the {len(both)} ranges both replayed: {len(before):,} → {len(after):,} releases')
+        names = lambda rs: collections.Counter((r['group'], r['name']) for r in rs.values())
+        a, b = names(before), names(after)
+        moved = sorted(set(a) | set(b), key=lambda k: -abs(a[k] - b[k]))[:20]
+        for k in moved:
+            if a[k] != b[k]:
+                print(f'  {a[k]:>6} → {b[k]:<6} {k[1][:80]!r} ({k[0]})')
+
+
+if __name__ == '__main__':
+    command = sys.argv[1] if len(sys.argv) > 1 else 'report'
+    if command == 'capture':
+        capture(int(sys.argv[2]) if len(sys.argv) > 2 else 100)
+    elif command == 'replay':
+        replay()
+    elif command == 'report':
+        report()
+    else:
+        raise SystemExit(__doc__)

deploy/deploy.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+# Deploys voyager 2 (branch rebuild) on homelab at one commit. Installed as
+# ~/bin/deploy-usenet-rebuild (by hand: it doesn't update itself). Gitea CI
+# runs it through the forced SSH command of the deploy key, as
+# "deploy-rebuild <sha>" (~/bin/deploy-usenet-indexer passes that on); by
+# hand: deploy-usenet-rebuild <sha>. The code comes from the Gitea repo, not
+# from the CI job.
+set -eu
+
+REPO=https://gitea.exp.sforum.nl/daan/usenet-indexer.git
+BRANCH=rebuild
+ROOT=/mnt/bulk/usenet-rebuild
+CACHE="$ROOT/.cache"
+GIT="$CACHE/repo.git"
+
+SHA="${1:-}"
+case "$SHA" in
+  *[!0-9a-f]* | "") echo "usage: deploy-usenet-rebuild <40-character commit sha>" >&2; exit 2 ;;
+esac
+[ "${#SHA}" -eq 40 ] || { echo "usage: deploy-usenet-rebuild <40-character commit sha>" >&2; exit 2; }
+
+mkdir -p "$CACHE"
+# One deploy at a time; a second one waits.
+exec 9>"$CACHE/deploy.lock"
+flock 9
+
+echo "== fetching $SHA"
+[ -d "$GIT" ] || git clone --quiet --bare "$REPO" "$GIT"
+git -C "$GIT" fetch --quiet --prune "$REPO" '+refs/heads/*:refs/heads/*'
+git -C "$GIT" cat-file -e "$SHA^{commit}"
+git -C "$GIT" merge-base --is-ancestor "$SHA" "refs/heads/$BRANCH" || { echo "$SHA is not on $BRANCH" >&2; exit 2; }
+
+STAGE=$(mktemp -d)
+trap 'rm -rf "$STAGE"' EXIT
+git -C "$GIT" archive "$SHA" | tar -x -C "$STAGE"
+rsync -a --delete "$STAGE"/ "$ROOT/src/"
+install -m 644 "$ROOT/src/deploy/compose.yaml" "$ROOT/compose.yaml"
+
+cd "$ROOT"
+echo "== building images"
+docker compose build migrate edge
+echo "== running migrations"
+docker compose run --rm --no-deps migrate
+# Only the app services, never postgres. The worker first: one holding a
+# stream keeps the node API from exiting.
+echo "== recreating app services"
+docker compose stop edge
+if [ -e "$CACHE/paused" ]; then
+  # Paused for the night (~/bin/usenet-rebuild-pause.sh): the workers stay stopped, the resume starts the new build.
+  echo "== paused: coordinator, prober, merger, judge, spotter, spot-inspector and edge left stopped"
+  docker compose up -d --no-deps node-api web
+else
+  docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
+  docker compose up -d --no-deps edge
+fi
+echo "$SHA $(date -Is)" >> "$ROOT/deployed.log"
+# The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor).
+docker image prune -f --filter label=com.docker.compose.project=usenet-rebuild >/dev/null
+docker compose ps --format '{{.Service}} {{.Status}}' node-api web coordinator prober merger judge spotter spot-inspector edge
+echo "== deployed $SHA"

deploy/usenet-rebuild-pause.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Installed as ~/bin/usenet-rebuild-pause.sh (by hand). Nightly (homelab crontab, 23:45): voyager 2's processing paused for the night; usenet-rebuild-resume.sh (07:45)
+# starts it again. Postgres, web and node-api stay up. Takes the deploy lock, so it never runs during a CI deploy, and
+# leaves .cache/paused, so a deploy in the night doesn't start the workers again.
+cd /mnt/bulk/usenet-rebuild || exit 1
+exec 9>.cache/deploy.lock
+flock 9
+touch .cache/paused
+date "+%F %T pausing"
+docker compose stop edge
+docker compose stop coordinator prober merger judge spotter spot-inspector
+docker compose ps --format "{{.Service}} {{.Status}}"

deploy/usenet-rebuild-resume.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Installed as ~/bin/usenet-rebuild-resume.sh (by hand). Nightly (homelab crontab, 07:45): voyager 2's processing back after usenet-rebuild-pause.sh (23:45). Takes the
+# deploy lock, like the pause.
+cd /mnt/bulk/usenet-rebuild || exit 1
+exec 9>.cache/deploy.lock
+flock 9
+rm -f .cache/paused
+date "+%F %T resuming"
+docker compose up -d --no-deps coordinator prober merger judge spotter spot-inspector
+docker compose up -d --no-deps edge
+docker compose ps --format "{{.Service}} {{.Status}}"

edge/Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.5"
@@ -32,6 +38,12 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -47,6 +59,12 @@ version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
+[[package]]
+name = "bytes"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+
 [[package]]
 name = "cc"
 version = "1.6.0"
@@ -93,6 +111,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crypto-common"
 version = "0.1.7"
@@ -114,10 +141,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "edge-bench-rs"
+name = "edge"
 version = "0.1.0"
 dependencies = [
- "base64",
+ "base64 0.22.1",
  "chrono",
  "flate2",
  "libdeflater",
@@ -126,10 +153,24 @@ dependencies = [
  "mimalloc",
  "regex",
  "rustc-hash",
+ "serde",
+ "serde_json",
  "sha1",
+ "signal-hook",
+ "ureq",
  "zstd",
 ]
 
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "find-msvc-tools"
 version = "0.1.14"
@@ -142,6 +183,8 @@ version = "1.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
 dependencies = [
+ "crc32fast",
+ "miniz_oxide",
  "zlib-rs",
 ]
 
@@ -179,6 +222,17 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.4.3"
@@ -190,6 +244,22 @@ dependencies = [
  "r-efi",
 ]
 
+[[package]]
+name = "http"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.65"
@@ -214,13 +284,19 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
 [[package]]
 name = "jobserver"
 version = "0.1.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
 dependencies = [
- "getrandom",
+ "getrandom 0.4.3",
  "libc",
 ]
 
@@ -299,6 +375,16 @@ dependencies = [
  "libmimalloc-sys",
 ]
 
+[[package]]
+name = "miniz_oxide"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -314,6 +400,12 @@ version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.17"
@@ -379,18 +471,110 @@ version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.17",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
+[[package]]
+name = "rustls"
+version = "0.23.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
+dependencies = [
+ "log",
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
 
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.6",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.7"
@@ -408,12 +592,44 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
 
+[[package]]
+name = "signal-hook"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
+dependencies = [
+ "errno",
+ "libc",
+]
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
 [[package]]
 name = "syn"
 version = "2.0.119"
@@ -448,12 +664,59 @@ version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "ureq"
+version = "3.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc"
+dependencies = [
+ "base64 0.23.1",
+ "flate2",
+ "log",
+ "percent-encoding",
+ "rustls",
+ "rustls-pki-types",
+ "ureq-proto",
+ "utf8-zero",
+ "webpki-roots",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be"
+dependencies = [
+ "base64 0.23.1",
+ "http",
+ "httparse",
+ "log",
+]
+
+[[package]]
+name = "utf8-zero"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.129"
@@ -499,6 +762,15 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "webpki-roots"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
+dependencies = [
+ "rustls-pki-types",
+]
+
 [[package]]
 name = "windows-core"
 version = "0.62.2"
@@ -558,12 +830,106 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "zeroize"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
+
 [[package]]
 name = "zlib-rs"
 version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
 
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+
 [[package]]
 name = "zstd"
 version = "0.13.3"

edge/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "edge"
+version = "0.1.0"
+edition = "2021"
+description = "Voyager 2's edge worker (edge-node) and its benchmark (edge-bench): the RangeIndexer in Rust, kept identical to lib/edge (check.ts)"
+
+[[bin]]
+name = "edge-bench"
+path = "src/main.rs"
+
+[[bin]]
+name = "edge-node"
+path = "src/bin/edge-node.rs"
+
+[dependencies]
+base64 = "0.22"
+chrono = "0.4"
+flate2 = { version = "1", default-features = false, features = ["zlib-rs"] }
+libdeflater = "1.26.1"
+md-5 = "0.10"
+memchr = "2.8.3"
+mimalloc = "0.1.52"
+regex = "1"
+rustc-hash = "2"
+serde = { version = "1.0.229", features = ["derive"] }
+serde_json = "1.0.151"
+sha1 = "0.10"
+signal-hook = "0.4.5"
+ureq = "3.4.2"
+zstd = "0.13"
+
+[profile.release]
+lto = true
+codegen-units = 1

edge/README.md

@@ -0,0 +1,238 @@
+# edge: voyager 2's edge worker (Rust)
+
+The product: `edge-node` (`src/bin/edge-node.rs`) is the worker that runs
+on homelab (compose service `edge`, image target `edge`). It leases ranges
+from the node API, reads their prefetched XOVER chunks, finds releases and
+posts batches. `edge-bench` (`src/main.rs`) replays a captured range through
+the same indexing (`src/lib.rs`) for timing and for comparing.
+
+The TypeScript `RangeIndexer` (`lib/edge`, `node ace edge:node`) is the
+reference: `check.ts` runs both on a capture and compares every output field
+by field. An identity or rule change goes into both and has to come out
+identical. CI (`.gitea/workflows/deploy.yaml`, job `edge`) runs `cargo test`
+and `check-fixtures.sh`, which compares both on the committed captures in
+`fixtures/` (25k overview lines each of a.b.models, a.b.ath and
+a.b.boneless, in 5 chunks); a push to `rebuild` deploys only when they
+match.
+
+```sh
+cargo test --release --manifest-path edge/Cargo.toml
+edge/check-fixtures.sh                       # from the repository root
+```
+
+In production since 2026-10-09: 122M headers in 10 minutes with 0.37 cores
+and 565 MB, next to the TypeScript workers' 24M with 0.61 cores and 11.3 GB
+(8 threads vs 8 processes): about 8× less CPU per header and 20× less
+memory. See "The worker" below.
+
+It started as a measurement (moved from `tools/edge-bench-rs` on
+2026-10-11): whether a Rust worker would be worth a second implementation.
+The findings below are from then.
+
+## Findings (2026-10-09)
+
+Same machine (the workstation, otherwise idle), same captures, 2M headers
+each:
+
+| Capture | TypeScript (`edge:bench`) | Rust (`edge-bench`) | Less CPU |
+|---|---|---|---|
+| a.b.boneless 2026 (160950000000–160951999999) | 11.1 CPU-s (~215k headers/s), 0.70 GB peak | 2.9 CPU-s (~690k/s), 0.24 GB | 3.8× |
+| a.b.boneless 2026 (160952000000–160953999999) | 11.8 CPU-s (~205k/s), 0.70 GB | 3.1 CPU-s (~640k/s), 0.24 GB | 3.8× |
+| a.b.ath 2013 (2549992990–2551992989) | 7.5 CPU-s (~320k/s), 0.74 GB | 2.15 CPU-s (~930k/s), 0.20 GB | 3.5× |
+| a.b.models (2017215–4017214; image sets, re-uploads) | 16.7 CPU-s (~139k/s) | 5.0 CPU-s (~400k/s) | 3.3× |
+
+**Comparing the numbers:**
+- **CPU seconds are the fair comparison.** `edge:bench`'s headers/s leaves
+  out the time Node's GC threads use; the Rust binary is single-threaded, so
+  its wall time is its CPU time.
+- **Homelab's cores** (Xeon E5-1650 v4) run the TypeScript ~10% slower than
+  the workstation's.
+
+**The Rust optimizations,** measured on boneless 2026 (all checked identical
+after each step):
+
+| Step | Headers/s |
+|---|---|
+| Straight port: regex crate for the patterns, chrono for dates, zlib-rs, std allocator | 360k |
+| File stems and subject remainders by hand: their 12 + 5 regexes were most of a new file's description (333k per boneless range, many of them per-article random subjects) | |
+| mimalloc for the many small strings | 495k |
+| Upload tools, release-name checks and the common Date formats by hand | 500k |
+| libdeflate: inflates a chunk whole in half of zlib's time (1.09 → 0.51 s per range) | |
+| memchr (SIMD) for line ends and tabs | ~700k |
+| Fewer allocations in `describe` (hashing the parts without joining them): no measurable gain, reverted | |
+
+Rows without a figure weren't measured on their own; each figure covers the
+rows above it.
+
+**What's left** is spread out: scanning and building segments, two sha1s per
+new file, aggregation, cut-off and chunk encoding. `perf` isn't available on
+the workstation (`perf_event_paranoid` = 3), so these come from coarse
+timers. The binary prints them as "of which aggregate …, evict+cutoff …,
+judge+encode …" on stderr.
+
+**What it means for the full target** (~1.25M articles/s with 50 NNTP
+connections):
+- **Worker cores:** ~1.5–2 instead of ~6–7. Spreading workers over other
+  machines (batcave, poweredge-kantoor) becomes mostly unnecessary.
+- **Total speed:** unchanged. The account's NNTP connections set it (~30k
+  articles/s each), not the worker.
+- **The cost:** a second implementation of `lib/headers`, `lib/release`,
+  `lib/rules`, `lib/files` (aggregation, volumes) and the chunk format.
+  Every rule change has to land in both and be checked identical (below)
+  for as long as both exist.
+
+## Running it
+
+You need a captured range. `edge:bench --capture` saves a range's compressed
+XOVER chunks once (it needs `NNTP_*` in the environment and one NNTP
+connection):
+
+```sh
+node ace edge:bench --capture --dir /path/to/capture --group alt.binaries.boneless --start 160950000000
+# 2M articles in chunks of 50k (--articles, --chunk), plus range.json
+```
+
+Benchmark both on it:
+
+```sh
+node ace edge:bench --dir /path/to/capture        # TypeScript: headers/s, CPU, inflate/index, peak RSS
+cd edge && cargo build --release
+./target/release/edge-bench /path/to/capture   # Rust: the same summary
+```
+
+Both take results every 10 chunks (`--batch-chunks`) and give a file up after
+5 idle chunks (`--idle`). Keep these equal on both sides when comparing.
+The rules are `DEFAULT_RULES` with only the range's group indexed and no
+dropped signatures, as in `edge:bench`.
+
+The two "kept: …" summary lines should already agree: release chunks,
+files, segments, payload size, candidates, fragments and samples. The
+summary can hide compensating differences, so compare the outputs
+themselves.
+
+## Comparing the output
+
+```sh
+./target/release/edge-bench /path/to/capture --dump /tmp/out.jsonl
+cd .. && node --import=@poppinss/ts-exec edge/check.ts /path/to/capture /tmp/out.jsonl
+```
+
+`--dump` writes every output of every batch as JSON lines:
+
+| Line `t` | Contents |
+|---|---|
+| `b` | batch boundaries: from, next, headers |
+| `r` | releases: every ReleaseChunk field, the payload in base64 |
+| `s` | samples: dropped releases kept in full, with their rule |
+| `c` | probe candidates |
+| `n` | rule counts |
+| `f` | fragments |
+
+`check.ts` runs the TypeScript `RangeIndexer` (from `lib/edge`, not a copy)
+on the same capture with the same batching. It matches items by batch and
+key: release key, candidate (release, kind, message-id), rule and key,
+fragment file key. Then it compares them field by field:
+
+```
+same batches: 4 TypeScript, 4 Rust
+same releases: 1055 TypeScript, 1055 Rust
+same samples: 2 TypeScript, 2 Rust
+same candidates: 1090 TypeScript, 1090 Rust
+same rule counts: 34 TypeScript, 34 Rust
+same fragments: 130 TypeScript, 130 Rust
+payload bytes: 13728351 TypeScript, 13727247 Rust
+```
+
+It exits 0 only when everything matches. Otherwise each kind says how many
+items are only on one side or differ, and shows the first differing fields.
+
+- **Payloads are compared decoded.** Both sides go through `decodeChunk`
+  from `lib/chunks`, and their files are compared, ordered by name and
+  subject. Node's built-in zstd and the zstd crate frame the same data a
+  little differently, so the compressed sizes differ by about 0.1%. The
+  checker prints them, but they don't count as a difference. Decoding the
+  Rust payloads with the TypeScript decoder also checks the Rust encoder.
+- **Order doesn't count.** Order within a batch isn't compared (the two
+  hash maps iterate differently). Grouping into batches, the order of a
+  release's files and the order of a fragment's segments are compared,
+  because they're part of the values.
+
+Run it after any change on either side. All four captures above are
+identical as of cfe85ab. The captures are kept on homelab in
+`/mnt/bulk/usenet-rebuild/bench/` (boneless-2026, boneless-2026b,
+ath-2013, models-a). models-a is a bottom-500 group (rank 501–1000 by
+span) with what the other three lack: image sets (38,630 release chunks
+for 412k files), documents, and re-uploads (521 chunks of a second or
+later upload of a key in a batch). Its chunk 26 is the plain fetch of a
+range whose compressed reply Eweka cuts short (LOG.md 2026-10-09 (11)).
+
+## JavaScript semantics the port keeps (`src/js.rs`, `src/names.rs`)
+
+Differences found with `check.ts` or the tests, kept as JavaScript has
+them:
+
+- **Regexes without the `u` flag:** `\d` is [0-9], `\b` an ASCII word
+  boundary, `/i` ASCII-only. They run as `(?-u)` byte regexes, or by hand.
+  The hand-written versions (file stems, subject remainders, upload tools,
+  release-name and file-counter checks) are tested against the regex
+  versions, which are kept for that (`*_re`).
+- **JS `\s` and `trim()`** include Unicode spaces (NBSP, U+2000–200A,
+  U+3000, BOM…); `split(/\s+/)` keeps leading and trailing empty words.
+- **`lastIndexOf(c, position)`** finds a match at or before the position
+  (the owner rule, the counter and filename scans). **`.length`** is in
+  UTF-16 units (`len16`). **`parseInt`** reads leading digits.
+- **`Date.parse`:** chrono follows RFC 5322 and rejects "UTC", which
+  `Date.parse` takes. It ends about a quarter of boneless's Date headers, so
+  it's mapped to GMT; check.ts found this. The common formats are parsed by
+  hand:
+
+  ```
+  [Day, ]D[D] Mon YY[YY] HH:MM:SS (GMT|UTC|UT|±hhmm)
+  ```
+
+  They ignore the weekday as `Date.parse` does (chrono rejects a weekday
+  that doesn't fit). Anything else goes to chrono. Rare forms `Date.parse`
+  takes but chrono doesn't would show up in check.ts as a `postedAt`
+  difference.
+- **Invalid UTF-8** becomes U+FFFD on both sides (`toString('utf8')`,
+  `from_utf8_lossy`).
+
+## Files
+
+| File | Contents |
+|---|---|
+| `src/main.rs` | arguments, the replay loop, timing, `--dump` |
+| `src/indexer.rs` | scan.ts (SegmentScanner, scanOverviewText), aggregator.ts, range_indexer.ts |
+| `src/names.rs` | lib/release (identity, random), lib/files/volumes, lib/rules (upload tools, judge) |
+| `src/codec.rs` | lib/chunks encodeChunk, format v2 |
+| `src/js.rs` | JavaScript semantics, sha1 |
+| `check.ts` | the comparison |
+
+## The worker
+
+```sh
+cargo build --release
+COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M]
+```
+
+- **One range per thread** (default one per core): one thread reads the
+  coordinator's stream and inflates a frame ahead, another posts the
+  batches in order (one may wait), the indexer goes on meanwhile.
+- **As `commands/edge_node.ts`:** retries for network trouble and 5xx;
+  409 means the lease is lost (the range stops, isn't given back); a
+  batch once 500k articles or 100k files wait (a picture group's batches
+  went over the server's 256 MB limit); a chunk that doesn't inflate is
+  reported (the coordinator fetches it plain next time); SIGTERM/SIGINT
+  give each range back after its current chunk; `--max-mbit` is the
+  node's download cap, shared by its threads. Plus a heartbeat when no
+  batch went out for a minute.
+- **Wire format:** `src/wire.rs` writes a batch as `toWire` does.
+  `edge-bench --wire` and `check.ts --wire` compare it with the
+  TypeScript's, field by field.
+- **End to end:** both workers indexed the a.b.models capture (spooled
+  as a range) through the web server; what the server stored was the
+  same (releases with every column, chunks, candidates, rule counts,
+  fragments with their JSON).
+- **On homelab:** compose service `edge` (deploy/compose.yaml), the
+  `edge` image target of the Dockerfile, `--threads $EDGE_THREADS` (8);
+  it talks to `node-api`.

edge/check-fixtures.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+# The Rust and TypeScript RangeIndexers on the committed captures (edge/fixtures: 25k
+# overview lines each of a.b.models, a.b.ath and a.b.boneless, in 5 chunks), compared
+# by check.ts field by field: exits non-zero on any difference. CI runs it; run it from
+# the repository root after any change to either side.
+set -eu
+cd "$(dirname "$0")/.."
+cargo build --release --quiet --manifest-path edge/Cargo.toml --bin edge-bench
+out=$(mktemp -d)
+trap 'rm -rf "$out"' EXIT
+status=0
+for capture in edge/fixtures/*/; do
+  name=$(basename "$capture")
+  edge/target/release/edge-bench "$capture" --batch-chunks 2 --idle 2 --dump "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" >/dev/null 2>&1
+  echo "== $name"
+  node --import=@poppinss/ts-exec edge/check.ts "$capture" "$out/$name.jsonl" --wire "$out/$name.wire.jsonl" --batch-chunks 2 --idle 2 || status=1
+done
+exit $status