Corpus: store chunks without the lines the worker only counts

History fetched ahead into the corpus (bulk HDD) is ~60 TB raw for the
1,180B missing articles, and the indexer keeps only ~38% of headers.
With CORPUS_FILTER (default on), Corpus.write leaves out the lines the
edge worker drops on their own, decided per line as RangeIndexer.line
does: a crosspost another indexed group owns, and a subject that names no
file part (~48% of the indexed ranges' headers). Release-level rules
(garbage, per-article, executable, poster) judge files across chunks and
aren't applied. The chunk is compressed again as zlib (the worker takes
it as before); beside it a .drops (gzip JSON) keeps the rule version,
the owners' priorities and per rule the article numbers (as differences,
crossposts by owner), and range.json says filtered. A test shows the
worker makes the same releases, candidates and fragments of a filtered
chunk. corpus:filter filters chunks stored raw (planned capture ranges;
--all for every range).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/unit/edge/corpus_filter.spec.ts

@@ -0,0 +1,83 @@
+import { test } from '@japa/runner'
+import { mkdtemp, readFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { deflateSync, gunzipSync, inflateSync } from 'node:zlib'
+import coordinatorConfig from '#config/coordinator'
+import Corpus from '#services/corpus'
+import { RangeIndexer, filterOverviews, undelta } from '#lib/edge/index'
+import { DEFAULT_RULES } from '#lib/rules/index'
+
+const GROUP = 'alt.binaries.test'
+const priorities = { 'alt.binaries.boneless': 1, [GROUP]: 2, 'alt.binaries.x': 3 }
+
+/** An overview line as XOVER sends it (number, subject, from, date, message-id, references, bytes, lines, Xref). */
+const line = (n: number, subject: string, xref = `news.example ${GROUP}:${n}`, messageId = `<a${n}@test>`) =>
+  [n, subject, 'poster <p@x>', 'Mon, 05 Oct 2026 10:00:00 +0000', messageId, '', '700000', '5000', `Xref: ${xref}`].join('\t')
+
+const LINES = [
+  line(100, 'Some.Show.S01E01 [1/2] - "Some.Show.S01E01.1080p.part1.rar" yEnc (1/2)'),
+  // Owned by boneless (priority 1): the worker counts it there.
+  line(101, 'Other [1/1] - "Other.mkv" yEnc (1/1)', `news.example ${GROUP}:101 alt.binaries.boneless:9`),
+  line(102, 'Some.Show.S01E01 [1/2] - "Some.Show.S01E01.1080p.part1.rar" yEnc (2/2)'),
+  // No file part in the subject.
+  line(103, 'obfuscated'),
+  // Also in a lower-priority group (3): ours.
+  line(104, 'Some.Show.S01E01 [2/2] - "Some.Show.S01E01.1080p.par2" yEnc (1/1)', `news.example ${GROUP}:104 alt.binaries.x:7`),
+  line(105, 'Pq4ZmXc8VbN2LkJh6GfDs3'),
+  // Without a message-id: the worker skips it uncounted, so it stays.
+  line(106, 'x', undefined, ''),
+]
+const TEXT = LINES.map((l) => `${l}\r\n`).join('')
+
+function index(text: string) {
+  const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities }, edgeArticles: 10 })
+  indexer.addText(text, 1001)
+  indexer.finish()
+  return indexer.take()
+}
+
+test.group('edge / corpus filter', () => {
+  test('leaves out crossposts another group owns and obfuscated subjects, keeps the rest byte for byte', ({ assert }) => {
+    const { kept, drops } = filterOverviews(TEXT, GROUP, priorities, 7)
+    assert.equal(kept, [LINES[0], LINES[2], LINES[4], LINES[6]].map((l) => `${l}\r\n`).join(''))
+    assert.equal(drops.ruleVersion, 7)
+    assert.equal(drops.lines, 7)
+    assert.equal(drops.kept, 4)
+    assert.deepEqual(Object.keys(drops.crosspost), ['alt.binaries.boneless'])
+    assert.deepEqual(undelta(drops.crosspost['alt.binaries.boneless']), [101])
+    assert.deepEqual(undelta(drops.obfuscated), [103, 105])
+    assert.deepEqual(drops.priorities, { 'alt.binaries.boneless': 1, [GROUP]: 2 })
+  })
+
+  test('the worker makes the same releases, candidates and fragments of the filtered chunk', ({ assert }) => {
+    const raw = index(TEXT)
+    const filtered = index(filterOverviews(TEXT, GROUP, priorities, 7).kept)
+    assert.isAbove(raw.releases.length, 0)
+    assert.deepEqual(filtered.releases, raw.releases)
+    assert.deepEqual(filtered.candidates, raw.candidates)
+    assert.deepEqual(filtered.fragments, raw.fragments)
+    // Only the counts of what was left out differ.
+    const without = (b: typeof raw) => b.counts.filter((c) => c.rule !== 'crosspost' && c.rule !== 'obfuscated')
+    assert.deepEqual(without(filtered), without(raw))
+  })
+
+  test('the corpus stores a filtered chunk as zlib with its .drops, and says so in range.json', async ({ assert, cleanup }) => {
+    const dir = await mkdtemp(join(tmpdir(), 'corpus-'))
+    const before = coordinatorConfig.corpus.dir
+    coordinatorConfig.corpus.dir = dir
+    cleanup(async () => {
+      coordinatorConfig.corpus.dir = before
+      await rm(dir, { recursive: true, force: true })
+    })
+    const range = { group: GROUP, start: 1, end: 100_000 }
+    const span = { from: 1, to: 100_000 }
+    await Corpus.write(range, span, deflateSync(Buffer.from(TEXT, 'latin1')), 1, { ruleVersion: 7, priorities })
+    const stored = (await Corpus.read(range, span))!
+    assert.equal(inflateSync(stored).toString('latin1'), filterOverviews(TEXT, GROUP, priorities, 7).kept)
+    const file = join(dir, GROUP, '1-100000', '0000-1-100000.gz')
+    const drops = JSON.parse(gunzipSync(await readFile(`${file}.drops`)).toString())
+    assert.deepEqual(undelta(drops.obfuscated), [103, 105])
+    assert.deepEqual(JSON.parse(await readFile(join(dir, GROUP, '1-100000', 'range.json'), 'utf8')), { ...range, filtered: ['crosspost', 'obfuscated'] })
+  })
+})