Corpus: stored XOVER chunks for the grouping sweeps

Ranges marked `capture` keep their XOVER chunks in CORPUS_DIR
(corpus/<group>/<start>-<end>/, edge-bench's layout, range.json once
complete; only chunks that inflate). The prefetcher reads a stored chunk
instead of fetching it, so a reset re-indexes them without the news
server; a chunk a worker reports corrupt goes from the corpus too.

edge-bench gets --releases (release chunks without payload, with their
first subjects) and ships in the edge image; deploy/corpus_sweep.py marks
more ranges (capture), replays the corpus with the deployed build (replay)
and reports suspects and the change since the previous build (report).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/corpus.ts

@@ -0,0 +1,72 @@
+import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { unzip } from 'node:zlib'
+import { promisify } from 'node:util'
+import coordinatorConfig from '#config/coordinator'
+import type { ChunkSpan } from '#services/spool'
+
+const inflate = promisify(unzip)
+
+export interface CorpusRange {
+  group: string
+  start: number
+  end: number
+}
+
+/**
+ * Stored XOVER chunks of the ranges marked `capture` (REBUILD.md, corpus):
+ * `<dir>/<group>/<start>-<end>/<nnnn>-<from>-<to>.gz` with a range.json once
+ * every chunk is there, the layout edge-bench reads (edge/fixtures). The
+ * grouping sweeps replay them with each build's rules instead of waiting for
+ * a re-index; the prefetcher reads a stored chunk instead of fetching it
+ * again. Off without CORPUS_DIR.
+ */
+class Corpus {
+  get dir(): string | undefined {
+    return coordinatorConfig.corpus.dir
+  }
+
+  private rangeDir(range: CorpusRange) {
+    return join(this.dir!, range.group, `${range.start}-${range.end}`)
+  }
+
+  private path(range: CorpusRange, span: ChunkSpan) {
+    const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
+    return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
+  }
+
+  /** A stored chunk, or null. */
+  async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
+    if (!this.dir) return null
+    return readFile(this.path(range, span)).catch(() => null)
+  }
+
+  /**
+   * Stores a chunk that inflates (a reply cut short isn't kept), and the
+   * range.json once the range is complete (`total` chunks).
+   */
+  async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
+    if (!this.dir) return
+    try {
+      await inflate(data)
+    } catch {
+      return
+    }
+    const dir = this.rangeDir(range)
+    const file = this.path(range, span)
+    await mkdir(dir, { recursive: true })
+    await writeFile(`${file}.part`, data)
+    await rename(`${file}.part`, file)
+    const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
+    if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
+  }
+
+  /** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
+  async removeChunk(range: CorpusRange, span: ChunkSpan) {
+    if (!this.dir) return
+    await rm(this.path(range, span), { force: true })
+    await rm(join(this.rangeDir(range), 'range.json'), { force: true })
+  }
+}
+
+export default new Corpus()

app/services/leases.ts

@@ -2,6 +2,7 @@ import db from '@adonisjs/lucid/services/db'
 import coordinatorConfig from '#config/coordinator'
 import RuleSets from '#services/rule_sets'
 import Spool from '#services/spool'
+import Corpus from '#services/corpus'
 import type { Lease } from '#lib/edge/index'
 
 export class LeaseLost extends Error {}
@@ -66,7 +67,13 @@ class Leases {
   async corrupt(rangeId: number, nodeId: number, from: number) {
     const range = await this.leased(rangeId, nodeId)
     const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find((s) => s.from === from)
-    if (span) await Spool.removeChunk(rangeId, span)
+    if (span) {
+      await Spool.removeChunk(rangeId, span)
+      if (range.capture) {
+        const group = await db.from('groups').where('id', range.group_id).select('name').first()
+        await Corpus.removeChunk({ group: group.name, start: Number(range.start_id), end: Number(range.end_id) }, span)
+      }
+    }
     const plain: number[] = (range.plain_chunks ?? []).map(Number)
     // Reported once already: the compressed copy may have been in flight (or the prefetcher's targets not
     // refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.

app/services/prefetcher.ts

@@ -3,6 +3,7 @@ import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import NntpPool from '#services/nntp_pool'
 import Spool, { type ChunkSpan } from '#services/spool'
+import Corpus from '#services/corpus'
 import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
 
 interface Target {
@@ -14,6 +15,10 @@ interface Target {
   missing: ChunkSpan[]
   /** All of the range's chunks, from its start. */
   total: number
+  start: number
+  end: number
+  /** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
+  capture: boolean
 }
 
 /** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
@@ -69,6 +74,8 @@ export class Prefetcher {
   private freshStats() {
     return {
       chunks: 0,
+      /** Chunks read from the corpus instead of fetched. */
+      stored: 0,
       articles: 0,
       fetchMs: 0,
       hedged: 0,
@@ -141,15 +148,23 @@ export class Prefetcher {
           await client.group(target.group)
           return client.xoverRaw(span.from, span.to)
         }
-        let data: Buffer
+        const corpusRange = { group: target.group, start: target.start, end: target.end }
+        // A range in the corpus is read from disk (a reset re-indexes it without the news server).
+        let data = target.capture ? await Corpus.read(corpusRange, span) : null
+        const stored = data !== null
         try {
-          data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
+          if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
         } finally {
           this.fetchingNow--
         }
-        this.stats.chunks++
-        this.stats.articles += span.to - span.from + 1
-        this.stats.fetchMs += Date.now() - fetchedAt
+        if (stored) {
+          this.stats.stored++
+        } else {
+          this.stats.chunks++
+          this.stats.articles += span.to - span.from + 1
+          this.stats.fetchMs += Date.now() - fetchedAt
+          if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
+        }
         const writtenAt = Date.now()
         this.writingNow++
         try {
@@ -288,7 +303,8 @@ export class Prefetcher {
       logger.info(
         `Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
           `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
-          `hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}.`
+          `hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
+          (this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus.` : '.')
       )
       const st = this.stats
       const avg = (n: number) => (samples > 0 ? n / samples : 0)
@@ -344,7 +360,7 @@ export class Prefetcher {
       this.spoolBytes = await Spool.size()
     }
     const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
-    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks'))
+    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
     // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
     const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
     // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
@@ -370,7 +386,9 @@ export class Prefetcher {
       for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
         if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
       }
-      if (missing.length > 0) targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total })
+      if (missing.length > 0) {
+        targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
+      }
     }
     return targets
   }