Corpus: store chunks without the lines the worker only counts

History fetched ahead into the corpus (bulk HDD) is ~60 TB raw for the
1,180B missing articles, and the indexer keeps only ~38% of headers.
With CORPUS_FILTER (default on), Corpus.write leaves out the lines the
edge worker drops on their own, decided per line as RangeIndexer.line
does: a crosspost another indexed group owns, and a subject that names no
file part (~48% of the indexed ranges' headers). Release-level rules
(garbage, per-article, executable, poster) judge files across chunks and
aren't applied. The chunk is compressed again as zlib (the worker takes
it as before); beside it a .drops (gzip JSON) keeps the rule version,
the owners' priorities and per rule the article numbers (as differences,
crossposts by owner), and range.json says filtered. A test shows the
worker makes the same releases, candidates and fragments of a filtered
chunk. corpus:filter filters chunks stored raw (planned capture ranges;
--all for every range).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/corpus_filter.ts

@@ -0,0 +1,54 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { access, readFile, readdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import db from '@adonisjs/lucid/services/db'
+import Corpus, { FILTERED } from '#services/corpus'
+import RuleSets from '#services/rule_sets'
+import coordinatorConfig from '#config/coordinator'
+
+/**
+ * Filters corpus chunks stored as the news server sent them (before CORPUS_FILTER, 2026-10-23): each chunk without a
+ * `.drops` of the planned capture ranges (--all: of every range in the corpus) written again as Corpus.write would.
+ */
+export default class CorpusFilterCommand extends BaseCommand {
+  static commandName = 'corpus:filter'
+  static description = 'Filters stored corpus chunks (planned capture ranges; --all: every range) as CORPUS_FILTER does'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'Every range in the corpus, also done ones (the grouping sweeps read them raw)' })
+  declare all: boolean
+
+  async run() {
+    const dir = coordinatorConfig.corpus.dir
+    if (!dir) return this.logger.error('CORPUS_DIR is not set')
+    const rules = await RuleSets.current()
+    const filter = { ruleVersion: rules.version, priorities: rules.priorities }
+    const query = db.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').where('r.capture', true).select('g.name as group', 'r.start_id', 'r.end_id')
+    const ranges = await (this.all ? query : query.where('r.status', 'planned'))
+    let chunks = 0
+    let before = 0
+    let after = 0
+    for (const r of ranges) {
+      const range = { group: r.group, start: Number(r.start_id), end: Number(r.end_id) }
+      const rangeDir = join(dir, range.group, `${range.start}-${range.end}`)
+      const files = await readdir(rangeDir).catch(() => [] as string[])
+      const gz = files.filter((f) => f.endsWith('.gz'))
+      for (const name of gz) {
+        if (files.includes(`${name}.drops`)) continue
+        const m = /^\d{4}-(\d+)-(\d+)\.gz$/.exec(name)
+        if (!m) continue
+        const data = await readFile(join(rangeDir, name))
+        await Corpus.write(range, { from: Number(m[1]), to: Number(m[2]) }, data, Number.POSITIVE_INFINITY, filter)
+        before += data.length
+        after += (await readFile(join(rangeDir, name))).length
+        chunks++
+      }
+      // A complete range's range.json says it's filtered now (Corpus.write only writes it at the last chunk).
+      const json = join(rangeDir, 'range.json')
+      if (gz.length > 0 && (await access(json).then(() => true, () => false))) await writeFile(json, JSON.stringify({ ...range, filtered: FILTERED }))
+      if (chunks > 0 && chunks % 1000 < gz.length) this.logger.info(`${chunks} chunks filtered: ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
+    }
+    this.logger.info(`Done: ${chunks} chunks of ${ranges.length} ranges filtered, ${(before / 1e9).toFixed(1)} → ${(after / 1e9).toFixed(1)} GB.`)
+  }
+}