Corpus: store chunks without the lines the worker only counts

History fetched ahead into the corpus (bulk HDD) is ~60 TB raw for the
1,180B missing articles, and the indexer keeps only ~38% of headers.
With CORPUS_FILTER (default on), Corpus.write leaves out the lines the
edge worker drops on their own, decided per line as RangeIndexer.line
does: a crosspost another indexed group owns, and a subject that names no
file part (~48% of the indexed ranges' headers). Release-level rules
(garbage, per-article, executable, poster) judge files across chunks and
aren't applied. The chunk is compressed again as zlib (the worker takes
it as before); beside it a .drops (gzip JSON) keeps the rule version,
the owners' priorities and per rule the article numbers (as differences,
crossposts by owner), and range.json says filtered. A test shows the
worker makes the same releases, candidates and fragments of a filtered
chunk. corpus:filter filters chunks stored raw (planned capture ranges;
--all for every range).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/edge/corpus_filter.ts

@@ -0,0 +1,78 @@
+import { SegmentScanner, scanOverviewText } from './scan.js'
+
+/**
+ * What a corpus chunk left out (REBUILD.md, corpus; LOG.md 2026-10-23 (101)): the overview lines the edge worker only
+ * counts and never indexes, decided per line as RangeIndexer.line does: a crosspost an indexed group with a lower
+ * priority number owns (by owner), and a subject that names no file part (obfuscated). Article numbers ascending,
+ * stored as differences. Release-level rules (garbage, per-article, executable, poster) judge files put together
+ * across chunks, so they aren't applied here.
+ */
+export interface ChunkDrops {
+  version: 1
+  /** The rule set (rule_sets.version) and group priorities the decision was made with. */
+  ruleVersion: number
+  priorities: Record<string, number>
+  /** Overview lines in the chunk, and kept. */
+  lines: number
+  kept: number
+  /** By owning group, article numbers as differences (the first from 0). */
+  crosspost: Record<string, number[]>
+  obfuscated: number[]
+}
+
+const deltas = (numbers: number[]) => numbers.map((n, i) => (i === 0 ? n : n - numbers[i - 1]))
+
+/** The article numbers back from ChunkDrops differences. */
+export function undelta(differences: number[]): number[] {
+  let at = 0
+  return differences.map((d) => (at += d))
+}
+
+/**
+ * A chunk's overview text without the lines the edge worker drops on their own, and what was dropped. Lines kept
+ * byte for byte (with their line ends); lines the worker skips uncounted (no number or message-id) are kept.
+ */
+export function filterOverviews(text: string, group: string, priorities: Record<string, number>, ruleVersion: number): { kept: string; drops: ChunkDrops } {
+  const scanner = new SegmentScanner()
+  const crosspost = new Map<string, number[]>()
+  const obfuscated: number[] = []
+  const dropped = new Set<number>()
+  let lines = 0
+  scanOverviewText(text, (n, subject, from, date, messageId, bytes, xref) => {
+    lines++
+    if (!Number.isFinite(n) || !messageId) return
+    const owner = xref ? scanner.owner(xref, group, priorities) : group
+    if (owner !== group) {
+      const list = crosspost.get(owner)
+      if (list) list.push(n)
+      else crosspost.set(owner, [n])
+      dropped.add(n)
+    } else if (!scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) {
+      obfuscated.push(n)
+      dropped.add(n)
+    }
+  })
+  const out: string[] = []
+  let start = 0
+  while (start < text.length) {
+    let end = text.indexOf('\n', start)
+    end = end === -1 ? text.length : end + 1
+    const tab = text.indexOf('\t', start)
+    const n = tab !== -1 && tab < end ? Number.parseInt(text.slice(start, tab), 10) : Number.NaN
+    if (!dropped.has(n)) out.push(text.slice(start, end))
+    start = end
+  }
+  const sorted = (list: number[]) => [...list].sort((a, b) => a - b)
+  return {
+    kept: out.join(''),
+    drops: {
+      version: 1,
+      ruleVersion,
+      priorities: Object.fromEntries(Object.entries(priorities).filter(([name]) => crosspost.has(name) || name === group)),
+      lines,
+      kept: lines - dropped.size,
+      crosspost: Object.fromEntries([...crosspost].map(([owner, list]) => [owner, deltas(sorted(list))])),
+      obfuscated: deltas(sorted(obfuscated)),
+    },
+  }
+}

lib/edge/index.ts

@@ -5,3 +5,4 @@ export type { Lease } from './lease.js'
 export { mergeFragments } from './merge.js'
 export type { EdgeSegment, FileMeta } from './scan.js'
 export { throttle } from './throttle.js'
+export { filterOverviews, undelta, type ChunkDrops } from './corpus_filter.js'