Corpus: store chunks without the lines the worker only counts

History fetched ahead into the corpus (bulk HDD) is ~60 TB raw for the
1,180B missing articles, and the indexer keeps only ~38% of headers.
With CORPUS_FILTER (default on), Corpus.write leaves out the lines the
edge worker drops on their own, decided per line as RangeIndexer.line
does: a crosspost another indexed group owns, and a subject that names no
file part (~48% of the indexed ranges' headers). Release-level rules
(garbage, per-article, executable, poster) judge files across chunks and
aren't applied. The chunk is compressed again as zlib (the worker takes
it as before); beside it a .drops (gzip JSON) keeps the rule version,
the owners' priorities and per rule the article numbers (as differences,
crossposts by owner), and range.json says filtered. A test shows the
worker makes the same releases, candidates and fragments of a filtered
chunk. corpus:filter filters chunks stored raw (planned capture ranges;
--all for every range).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/prefetcher.ts

@@ -3,7 +3,8 @@ import logger from '@adonisjs/core/services/logger'
 import coordinatorConfig from '#config/coordinator'
 import NntpPool from '#services/nntp_pool'
 import Spool, { type ChunkSpan } from '#services/spool'
-import Corpus from '#services/corpus'
+import Corpus, { type CorpusFilter } from '#services/corpus'
+import RuleSets from '#services/rule_sets'
 import CatalogSpace from '#services/catalog_space'
 import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
 
@@ -175,7 +176,7 @@ export class Prefetcher {
           this.stats.chunks++
           this.stats.articles += span.to - span.from + 1
           this.stats.fetchMs += Date.now() - fetchedAt
-          if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
+          if (target.capture) await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
         }
         if (target.corpusOnly) {
           this.stats.captured++
@@ -415,6 +416,18 @@ export class Prefetcher {
     return targets
   }
 
+  private filterRules: { at: number; filter: CorpusFilter } | null = null
+
+  /** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
+  private async corpusFilter(): Promise<CorpusFilter | undefined> {
+    if (!coordinatorConfig.corpus.filter) return undefined
+    if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
+      const rules = await RuleSets.current()
+      this.filterRules = { at: Date.now(), filter: { ruleVersion: rules.version, priorities: rules.priorities } }
+    }
+    return this.filterRules.filter
+  }
+
   /**
    * Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus; as many of a
    * group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which took 4 of the