usenet-indexer/tools/edge-bench-rs/check.ts

/**
 * Compares edge-bench-rs's output (--dump) with the TypeScript RangeIndexer
 * on the same captured range, item by item: releases (every field, and the
 * payload decoded with lib/chunks), samples, probe candidates, rule counts,
 * fragments and batch boundaries.
 *
 *   node --import=@poppinss/ts-exec tools/edge-bench-rs/check.ts <dir> <dump.jsonl> [--batch-chunks 10] [--idle 5]
 */
import { readFile, readdir } from 'node:fs/promises'
import { join } from 'node:path'
import { unzipSync } from 'node:zlib'
import { isDeepStrictEqual } from 'node:util'
import { RangeIndexer, type EdgeBatch } from '#lib/edge/index'
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
import { decodeChunk } from '#lib/chunks/index'

const [dir, dumpPath] = process.argv.slice(2)
const flag = (name: string, fallback: number) => {
  const i = process.argv.indexOf(name)
  return i === -1 ? fallback : Number(process.argv[i + 1])
}
const batchChunks = flag('--batch-chunks', 10)
const idle = flag('--idle', 5)

const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] }
const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle })
const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()
const batches: EdgeBatch[] = []
for (const [i, file] of files.entries()) {
  const next = Number(file.split('-')[2].replace('.gz', '')) + 1
  indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next)
  const last = i === files.length - 1
  if (last) indexer.finish()
  if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take())
}

/** Canonical lines per kind, in a stable order, from both sides. */
type Item = Record<string, unknown>
const sortFiles = (decoded: ReturnType<typeof decodeChunk>) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1))
const payloadBytes = { ts: 0, rs: 0 }
let side: 'ts' | 'rs' = 'ts'
const release = (r: Item, b: number) => {
  const { payload, ...rest } = r as Item & { payload: Buffer }
  // The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little
  // differently); what the payload decodes to is.
  payloadBytes[side] += payload.length
  return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) }
}
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
batches.forEach((batch, b) => {
  ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers })
  for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
  for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
  for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
  for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
  for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
})

const rs: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
side = 'rs'
let b = -1
for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
  if (!line) continue
  const { t, ...item } = JSON.parse(line)
  if (t === 'b') b++
  if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
  else if (t === 's') rs.s.push({ b, key: item.key, upload: item.upload, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
  else rs[t].push({ b, ...item })
}

const keyOf: Record<string, (i: Item) => string> = {
  b: (i) => String(i.b),
  r: (i) => `${i.b} ${i.key} ${i.upload}`,
  s: (i) => `${i.b} ${i.key} ${i.upload}`,
  c: (i) => `${i.b} ${i.releaseKey} ${i.upload} ${i.kind} ${i.messageId}`,
  n: (i) => `${i.b} ${i.rule} ${i.key}`,
  f: (i) => `${i.b} ${i.fileKey}`,
}
const names: Record<string, string> = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' }
let failed = false
for (const kind of Object.keys(keyOf)) {
  const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i]))
  const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i]))
  const missing = [...a.keys()].filter((k) => !z.has(k))
  const extra = [...z.keys()].filter((k) => !a.has(k))
  const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k)))
  const ok = missing.length === 0 && extra.length === 0 && differ.length === 0
  failed ||= !ok
  console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`)
  for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(`  only one side: ${k}`)
  for (const k of differ.slice(0, 2)) {
    const x = a.get(k)!
    const y = z.get(k)!
    const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f]))
    console.log(`  ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`)
  }
}
console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`)
process.exit(failed ? 1 : 0)