usenet-indexer/tests/unit/split.spec.ts

import { test } from '@japa/runner'
import { splitInstances, splitVerified, verifyVolume, type SplitArticle } from '#lib/split/index'
import { crc32Hex } from '#lib/yenc/index'

/** Deterministic pseudo-random numbers. */
function rng(seed: number) {
  let x = seed
  return () => {
    x = (x * 1103515245 + 12345) & 0x7fffffff
    return x / 0x7fffffff
  }
}

interface PostOptions {
  volumes: number
  parts: number
  /** Parallel upload connections; they finish slightly out of order. */
  connections?: number
  /** First article number of the post. */
  start?: number
  /** Unrelated articles between two articles of this post (other posters). */
  noise?: number
  /** Fraction of articles that never arrive. */
  drop?: number
  label?: string
  seed?: number
}

/** Simulates a posting tool uploading equal-sized volumes. */
function post(opts: PostOptions): SplitArticle[] {
  const random = rng(opts.seed ?? 1)
  const connections = opts.connections ?? 1
  const label = opts.label ?? 'p'
  const queue: { volume: number; number: number }[] = []
  for (let v = 0; v < opts.volumes; v++) for (let n = 1; n <= opts.parts; n++) queue.push({ volume: v, number: n })

  const articles: SplitArticle[] = []
  let articleNumber = opts.start ?? 1000
  for (let i = 0; i < queue.length; i += connections) {
    // One round: each connection posts one article; they land in random order.
    const round = queue.slice(i, i + connections).sort(() => random() - 0.5)
    for (const { volume, number } of round) {
      articleNumber += 1 + Math.floor(random() * (opts.noise ?? 0))
      if (random() < (opts.drop ?? 0)) continue
      articles.push({ number, messageId: `${label}${volume}.${number}`, bytes: 700_000, articleNumber })
    }
  }
  return articles
}

/** Source volume of an article, from its simulated message-id. */
const source = (a: SplitArticle) => a.messageId.split('.')[0]

/** Every result volume must contain articles of one source volume only. */
function assertPure(assert: any, volumes: SplitArticle[][]) {
  for (const volume of volumes) {
    assert.lengthOf(new Set(volume.map(source)), 1, `mixed volume: ${volume.map((a) => a.messageId).join(' ')}`)
  }
}

test.group('split / splitInstances', () => {
  test('separates sequentially posted volumes', ({ assert }) => {
    const volumes = splitInstances(post({ volumes: 3, parts: 14 }), { totalParts: 14 })
    assert.lengthOf(volumes, 3)
    assertPure(assert, volumes)
    assert.isTrue(volumes.every((v) => v.length === 14))
  })

  test('separates volumes posted over interleaved connections, out of order', ({ assert }) => {
    for (const seed of [1, 2, 3, 4, 5]) {
      const volumes = splitInstances(post({ volumes: 6, parts: 40, connections: 8, noise: 30, seed }), { totalParts: 40 })
      assertPure(assert, volumes)
      assert.lengthOf(volumes, 6, `seed ${seed}`)
    }
  })

  test('a volume that completes is never mixed, even with heavy loss', ({ assert }) => {
    for (const seed of [11, 12, 13, 14, 15, 16, 17, 18]) {
      const articles = post({ volumes: 5, parts: 30, connections: 6, noise: 20, drop: 0.2, seed })
      const volumes = splitInstances(articles, { totalParts: 30 })
      assertPure(assert, volumes.filter((v) => v.length === 30))
    }
  })

  test('a missing article only affects its own volume', ({ assert }) => {
    const articles = post({ volumes: 4, parts: 20, connections: 4, drop: 0.05, seed: 7 })
    const volumes = splitInstances(articles, { totalParts: 20 })
    assertPure(assert, volumes)
    assert.lengthOf(volumes, 4)
  })

  test('posts of the same size at different times do not mix', ({ assert }) => {
    // An earlier post that lost articles (e.g. before the backfill window), then two more.
    const articles = [
      ...post({ volumes: 3, parts: 14, connections: 4, drop: 0.3, label: 'a', start: 1_000, seed: 3 }),
      ...post({ volumes: 3, parts: 14, connections: 4, label: 'b', start: 200_000, seed: 4 }),
      ...post({ volumes: 2, parts: 14, connections: 2, label: 'c', start: 900_000, seed: 5 }),
    ]
    const volumes = splitInstances(articles, { totalParts: 14 })
    // The damaged post may produce mixed volumes, but those stay incomplete
    // (they never become files); every volume that completes is pure.
    const complete = volumes.filter((v) => v.length === 14)
    assertPure(assert, complete)
    assert.includeMembers(complete.map((v) => source(v[0])), ['b0', 'b1', 'b2', 'c0', 'c1'])
    // The later, intact posts are not affected at all.
    assertPure(assert, volumes.filter((v) => !source(v[0]).startsWith('a')))
  })

  test('ignores duplicate message-ids and does not depend on input order', ({ assert }) => {
    const articles = post({ volumes: 2, parts: 5, connections: 3, seed: 9 })
    const shuffled = [...articles].reverse().concat(articles[0])
    assert.deepEqual(splitInstances(shuffled, { totalParts: 5 }), splitInstances(articles, { totalParts: 5 }))
  })
})

/** Like post(), with real data per volume so part and file CRCs are genuine. */
function postWithCrcs(opts: PostOptions): SplitArticle[] {
  const random = rng((opts.seed ?? 1) + 1000)
  const size = 64
  const data = new Map<string, Buffer>()
  const articles = post(opts)
  for (const a of articles) {
    const volume = source(a)
    if (!data.has(volume)) {
      data.set(volume, Buffer.from(Array.from({ length: opts.parts * size }, () => Math.floor(random() * 256))))
    }
    const file = data.get(volume)!
    a.pcrc32 = crc32Hex(file.subarray((a.number - 1) * size, a.number * size))
    a.partSize = size
    a.fileCrc32 = a.number === opts.parts ? crc32Hex(file) : null
  }
  return articles
}

test.group('split / verified', () => {
  test('verifyVolume accepts a correct volume and rejects a mixed one', ({ assert }) => {
    const articles = postWithCrcs({ volumes: 2, parts: 6 })
    const volumeA = articles.filter((a) => source(a) === 'p0')
    assert.isTrue(verifyVolume(volumeA, 6))
    const mixed = volumeA.map((a) => (a.number === 3 ? articles.find((b) => source(b) === 'p1' && b.number === 3)! : a))
    assert.isFalse(verifyVolume(mixed, 6))
    assert.isFalse(verifyVolume(volumeA.slice(0, 5), 6))
  })

  test('only verified volumes come out, and they are never mixed', ({ assert }) => {
    for (const seed of [21, 22, 23, 24, 25, 26]) {
      const articles = [
        ...postWithCrcs({ volumes: 4, parts: 30, connections: 16, noise: 20, drop: 0.05, label: 'a', seed }),
        ...postWithCrcs({ volumes: 3, parts: 30, connections: 16, noise: 20, label: 'b', start: 300_000, seed: seed + 50 }),
      ]
      for (const volume of splitVerified(articles, { totalParts: 30 })) {
        assert.isTrue(verifyVolume(volume, 30))
        assertPure(assert, [volume])
      }
    }
  })

  test('recovers every volume of a cleanly posted set', ({ assert }) => {
    const volumes = splitVerified(postWithCrcs({ volumes: 5, parts: 20, connections: 4, noise: 10, seed: 31 }), { totalParts: 20 })
    assert.lengthOf(volumes, 5)
  })
})