Corpus: stored XOVER chunks for the grouping sweeps

Ranges marked `capture` keep their XOVER chunks in CORPUS_DIR
(corpus/<group>/<start>-<end>/, edge-bench's layout, range.json once
complete; only chunks that inflate). The prefetcher reads a stored chunk
instead of fetching it, so a reset re-indexes them without the news
server; a chunk a worker reports corrupt goes from the corpus too.

edge-bench gets --releases (release chunks without payload, with their
first subjects) and ships in the edge image; deploy/corpus_sweep.py marks
more ranges (capture), replays the corpus with the deployed build (replay)
and reports suspects and the change since the previous build (report).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/prefetcher.ts

@@ -3,6 +3,7 @@ import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import NntpPool from '#services/nntp_pool'
 import Spool, { type ChunkSpan } from '#services/spool'
+import Corpus from '#services/corpus'
 import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
 
 interface Target {
@@ -14,6 +15,10 @@ interface Target {
   missing: ChunkSpan[]
   /** All of the range's chunks, from its start. */
   total: number
+  start: number
+  end: number
+  /** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
+  capture: boolean
 }
 
 /** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
@@ -69,6 +74,8 @@ export class Prefetcher {
   private freshStats() {
     return {
       chunks: 0,
+      /** Chunks read from the corpus instead of fetched. */
+      stored: 0,
       articles: 0,
       fetchMs: 0,
       hedged: 0,
@@ -141,15 +148,23 @@ export class Prefetcher {
           await client.group(target.group)
           return client.xoverRaw(span.from, span.to)
         }
-        let data: Buffer
+        const corpusRange = { group: target.group, start: target.start, end: target.end }
+        // A range in the corpus is read from disk (a reset re-indexes it without the news server).
+        let data = target.capture ? await Corpus.read(corpusRange, span) : null
+        const stored = data !== null
         try {
-          data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
+          if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
         } finally {
           this.fetchingNow--
         }
-        this.stats.chunks++
-        this.stats.articles += span.to - span.from + 1
-        this.stats.fetchMs += Date.now() - fetchedAt
+        if (stored) {
+          this.stats.stored++
+        } else {
+          this.stats.chunks++
+          this.stats.articles += span.to - span.from + 1
+          this.stats.fetchMs += Date.now() - fetchedAt
+          if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
+        }
         const writtenAt = Date.now()
         this.writingNow++
         try {
@@ -288,7 +303,8 @@ export class Prefetcher {
       logger.info(
         `Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
           `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
-          `hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}.`
+          `hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
+          (this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus.` : '.')
       )
       const st = this.stats
       const avg = (n: number) => (samples > 0 ? n / samples : 0)
@@ -344,7 +360,7 @@ export class Prefetcher {
       this.spoolBytes = await Spool.size()
     }
     const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
-    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks'))
+    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
     // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
     const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
     // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
@@ -370,7 +386,9 @@ export class Prefetcher {
       for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
         if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
       }
-      if (missing.length > 0) targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total })
+      if (missing.length > 0) {
+        targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
+      }
     }
     return targets
   }