Corpus: store chunks without the lines the worker only counts

History fetched ahead into the corpus (bulk HDD) is ~60 TB raw for the
1,180B missing articles, and the indexer keeps only ~38% of headers.
With CORPUS_FILTER (default on), Corpus.write leaves out the lines the
edge worker drops on their own, decided per line as RangeIndexer.line
does: a crosspost another indexed group owns, and a subject that names no
file part (~48% of the indexed ranges' headers). Release-level rules
(garbage, per-article, executable, poster) judge files across chunks and
aren't applied. The chunk is compressed again as zlib (the worker takes
it as before); beside it a .drops (gzip JSON) keeps the rule version,
the owners' priorities and per rule the article numbers (as differences,
crossposts by owner), and range.json says filtered. A test shows the
worker makes the same releases, candidates and fragments of a filtered
chunk. corpus:filter filters chunks stored raw (planned capture ranges;
--all for every range).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/corpus.ts

@@ -1,11 +1,22 @@
 import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
-import { unzip } from 'node:zlib'
+import { deflate as deflateCb, gzip as gzipCb, unzip } from 'node:zlib'
 import { promisify } from 'node:util'
 import coordinatorConfig from '#config/coordinator'
+import { filterOverviews } from '#lib/edge/index'
 import type { ChunkSpan } from '#services/spool'
 
 const inflate = promisify(unzip)
+const deflate = promisify(deflateCb)
+const gzip = promisify(gzipCb)
+
+/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
+export interface CorpusFilter {
+  ruleVersion: number
+  priorities: Record<string, number>
+}
+/** The rules a filtered chunk leaves out (lib/edge/corpus_filter.ts), in its range.json. */
+export const FILTERED = ['crosspost', 'obfuscated']
 
 export interface CorpusRange {
   group: string
@@ -19,7 +30,10 @@ export interface CorpusRange {
  * every chunk is there, the layout edge-bench reads (edge/fixtures). The
  * grouping sweeps replay them with each build's rules instead of waiting for
  * a re-index; the prefetcher reads a stored chunk instead of fetching it
- * again. Off without CORPUS_DIR.
+ * again. Off without CORPUS_DIR. Filtered (CORPUS_FILTER; LOG.md 2026-10-23
+ * (101)): the lines the edge worker only counts are left out, ~half of them,
+ * and each chunk's `.drops` beside it (gzip JSON, ChunkDrops) lists their
+ * article numbers by rule; its range.json says `filtered`.
  */
 class Corpus {
   get dir(): string | undefined {
@@ -61,28 +75,40 @@ class Corpus {
 
   /**
    * Stores a chunk that inflates (a reply cut short isn't kept), and the
-   * range.json once the range is complete (`total` chunks).
+   * range.json once the range is complete (`total` chunks). With `filter`,
+   * without the lines the edge worker drops on their own, compressed again
+   * as the news server's (zlib), with its `.drops` written first.
    */
-  async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
+  async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number, filter?: CorpusFilter) {
     if (!this.dir) return
+    let raw: Buffer
     try {
-      await inflate(data)
+      raw = await inflate(data)
     } catch {
       return
     }
     const dir = this.rangeDir(range)
     const file = this.path(range, span)
     await mkdir(dir, { recursive: true })
-    await writeFile(`${file}.part`, data)
+    let stored = data
+    if (filter) {
+      // Latin-1 both ways: every byte back as it was.
+      const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
+      stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
+      await writeFile(`${file}.drops.part`, await gzip(JSON.stringify(drops)))
+      await rename(`${file}.drops.part`, `${file}.drops`)
+    }
+    await writeFile(`${file}.part`, stored)
     await rename(`${file}.part`, file)
     const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
-    if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
+    if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(filter ? { ...range, filtered: FILTERED } : range))
   }
 
   /** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
   async removeChunk(range: CorpusRange, span: ChunkSpan) {
     if (!this.dir) return
     await rm(this.path(range, span), { force: true })
+    await rm(`${this.path(range, span)}.drops`, { force: true })
     await rm(join(this.rangeDir(range), 'range.json'), { force: true })
   }
 }

app/services/prefetcher.ts

@@ -3,7 +3,8 @@ import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import NntpPool from '#services/nntp_pool'
 import Spool, { type ChunkSpan } from '#services/spool'
-import Corpus from '#services/corpus'
+import Corpus, { type CorpusFilter } from '#services/corpus'
+import RuleSets from '#services/rule_sets'
 import CatalogSpace from '#services/catalog_space'
 import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
 
@@ -175,7 +176,7 @@ export class Prefetcher {
           this.stats.chunks++
           this.stats.articles += span.to - span.from + 1
           this.stats.fetchMs += Date.now() - fetchedAt
-          if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
+          if (target.capture) await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
         }
         if (target.corpusOnly) {
           this.stats.captured++
@@ -415,6 +416,18 @@ export class Prefetcher {
     return targets
   }
 
+  private filterRules: { at: number; filter: CorpusFilter } | null = null
+
+  /** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
+  private async corpusFilter(): Promise<CorpusFilter | undefined> {
+    if (!coordinatorConfig.corpus.filter) return undefined
+    if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
+      const rules = await RuleSets.current()
+      this.filterRules = { at: Date.now(), filter: { ruleVersion: rules.version, priorities: rules.priorities } }
+    }
+    return this.filterRules.filter
+  }
+
   /**
    * Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus; as many of a
    * group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which took 4 of the