usenet-indexer/edge/check.ts

/**
 * Compares edge-bench's output (--dump) with the TypeScript RangeIndexer
 * on the same captured range, item by item: releases (every field, and the
 * payload decoded with lib/chunks), samples, probe candidates, rule counts,
 * fragments and batch boundaries.
 *
 *   node --import=@poppinss/ts-exec edge/check.ts <dir> <dump.jsonl> [--wire batches.jsonl] [--batch-chunks 10] [--idle 5]
 *
 * With --wire (edge-bench --wire): also each batch as the Rust worker
 * posts it, against toWire of the TypeScript batch, field by field.
 */
import { readFile, readdir } from 'node:fs/promises'
import { join } from 'node:path'
import { unzipSync } from 'node:zlib'
import { isDeepStrictEqual } from 'node:util'
import { RangeIndexer, toWire, type EdgeBatch } from '#lib/edge/index'
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
import { decodeChunk } from '#lib/chunks/index'

const [dir, dumpPath] = process.argv.slice(2)
const flag = (name: string, fallback: number) => {
  const i = process.argv.indexOf(name)
  return i === -1 ? fallback : Number(process.argv[i + 1])
}
const batchChunks = flag('--batch-chunks', 10)
const idle = flag('--idle', 5)

const { group, start, end } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
const rules: Rules = { ...DEFAULT_RULES, priorities: { [group]: 1 }, droppedSignatures: [] }
const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: idle })
const files = (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()
const batches: EdgeBatch[] = []
for (const [i, file] of files.entries()) {
  const next = Number(file.split('-')[2].replace('.gz', '')) + 1
  indexer.addText(unzipSync(await readFile(join(dir, file))).toString('utf8'), next)
  const last = i === files.length - 1
  if (last) indexer.finish()
  if (last || (i + 1) % batchChunks === 0) batches.push(indexer.take())
}

/** Canonical lines per kind, in a stable order, from both sides. */
type Item = Record<string, unknown>
const sortFiles = (decoded: ReturnType<typeof decodeChunk>) => [...decoded].sort((a, b) => (a.name + a.subject < b.name + b.subject ? -1 : 1))
const payloadBytes = { ts: 0, rs: 0 }
let side: 'ts' | 'rs' = 'ts'
const release = (r: Item, b: number) => {
  const { payload, ...rest } = r as Item & { payload: Buffer }
  // The compressed size isn't compared (node's zstd and the zstd crate frame alike data a little
  // differently); what the payload decodes to is.
  payloadBytes[side] += payload.length
  return { b, ...rest, decoded: sortFiles(decodeChunk(payload)) }
}
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
batches.forEach((batch, b) => {
  ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers })
  for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
  for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
  for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
  for (const n of batch.counts) ts.n.push({ b, rule: n.rule, key: n.key, articles: n.articles, files: n.files })
  for (const f of batch.fragments) ts.f.push({ b, fileKey: f.fileKey, name: f.name, totalParts: f.totalParts, postedAt: f.postedAt, segments: f.segments })
})

const rs: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
side = 'rs'
let b = -1
for (const line of (await readFile(dumpPath, 'utf8')).split('\n')) {
  if (!line) continue
  const { t, ...item } = JSON.parse(line)
  if (t === 'b') b++
  if (t === 'r') rs.r.push(release({ ...item, payload: Buffer.from(item.payload, 'base64') }, b))
  else if (t === 's') rs.s.push({ b, key: item.key, upload: item.upload, rule: item.rule, ruleKey: item.ruleKey, segments: item.segments })
  else rs[t].push({ b, ...item })
}

const keyOf: Record<string, (i: Item) => string> = {
  b: (i) => String(i.b),
  r: (i) => `${i.b} ${i.key} ${i.upload}`,
  s: (i) => `${i.b} ${i.key} ${i.upload}`,
  c: (i) => `${i.b} ${i.releaseKey} ${i.upload} ${i.kind} ${i.messageId}`,
  n: (i) => `${i.b} ${i.rule} ${i.key}`,
  f: (i) => `${i.b} ${i.fileKey}`,
}
const names: Record<string, string> = { b: 'batches', r: 'releases', s: 'samples', c: 'candidates', n: 'rule counts', f: 'fragments' }
let failed = false
for (const kind of Object.keys(keyOf)) {
  const a = new Map(ts[kind].map((i) => [keyOf[kind](i), i]))
  const z = new Map(rs[kind].map((i) => [keyOf[kind](i), i]))
  const missing = [...a.keys()].filter((k) => !z.has(k))
  const extra = [...z.keys()].filter((k) => !a.has(k))
  const differ = [...a.keys()].filter((k) => z.has(k) && !isDeepStrictEqual(a.get(k), z.get(k)))
  const ok = missing.length === 0 && extra.length === 0 && differ.length === 0
  failed ||= !ok
  console.log(`${ok ? 'same' : 'DIFF'} ${names[kind]}: ${a.size} TypeScript, ${z.size} Rust${ok ? '' : `; ${missing.length} only in TypeScript, ${extra.length} only in Rust, ${differ.length} differ`}`)
  for (const k of [...missing.slice(0, 2), ...extra.slice(0, 2)]) console.log(`  only one side: ${k}`)
  for (const k of differ.slice(0, 2)) {
    const x = a.get(k)!
    const y = z.get(k)!
    const fields = Object.keys(x).filter((f) => !isDeepStrictEqual(x[f], y[f]))
    console.log(`  ${k}: ${fields.map((f) => `${f}: ${JSON.stringify(x[f]).slice(0, 160)} vs ${JSON.stringify(y[f]).slice(0, 160)}`).join('; ')}`)
  }
}
console.log(`payload bytes: ${payloadBytes.ts} TypeScript, ${payloadBytes.rs} Rust`)

const wireIndex = process.argv.indexOf('--wire')
if (wireIndex !== -1) {
  // Order within a batch doesn't count; payloads by what they decode to.
  const by = (f: (x: any) => string) => (a: any, b: any) => (f(a) < f(b) ? -1 : f(a) > f(b) ? 1 : 0)
  const normal = (w: any) => ({
    ...w,
    releases: w.releases.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
    samples: w.samples.map((r: any) => ({ ...r, payload: sortFiles(decodeChunk(Buffer.from(r.payload, 'base64'))) })).sort(by((r) => `${r.key} ${r.upload}`)),
    candidates: [...w.candidates].sort(by((c) => JSON.stringify(c))),
    counts: [...w.counts].sort(by((c) => `${c.rule} ${c.key}`)),
    fragments: [...w.fragments].sort(by((f) => `${f.fileKey} ${f.segments[0]?.[1]}`)),
  })
  const rust = (await readFile(process.argv[wireIndex + 1], 'utf8')).split('\n').filter(Boolean).map((l) => normal(JSON.parse(l)))
  const typescript = batches.map((b) => normal(JSON.parse(JSON.stringify(toWire(b)))))
  let wireSame = rust.length === typescript.length
  const firstDiff = (x: any, y: any, path: string): string | null => {
    if (isDeepStrictEqual(x, y)) return null
    if (typeof x !== 'object' || typeof y !== 'object' || x === null || y === null) return `${path}: ${JSON.stringify(x)?.slice(0, 120)} vs ${JSON.stringify(y)?.slice(0, 120)}`
    for (const k of new Set([...Object.keys(x), ...Object.keys(y)])) {
      const d = firstDiff(x[k], y[k], `${path}.${k}`)
      if (d) return d
    }
    return path
  }
  for (let i = 0; i < Math.max(rust.length, typescript.length); i++) {
    const d = firstDiff(typescript[i], rust[i], `batch ${i}`)
    if (d) {
      wireSame = false
      console.log(`DIFF wire: ${d}`)
      break
    }
  }
  if (wireSame) console.log(`same wire batches: ${typescript.length} TypeScript, ${rust.length} Rust`)
  failed ||= !wireSame
}
process.exit(failed ? 1 : 0)