import { test } from '@japa/runner' import { splitInstances, splitVerified, verifyVolume, type SplitArticle } from '#lib/split/index' import { crc32Hex } from '#lib/yenc/index' /** Deterministic pseudo-random numbers. */ function rng(seed: number) { let x = seed return () => { x = (x * 1103515245 + 12345) & 0x7fffffff return x / 0x7fffffff } } interface PostOptions { volumes: number parts: number /** Parallel upload connections; they finish slightly out of order. */ connections?: number /** First article number of the post. */ start?: number /** Unrelated articles between two articles of this post (other posters). */ noise?: number /** Fraction of articles that never arrive. */ drop?: number label?: string seed?: number } /** Simulates a posting tool uploading equal-sized volumes. */ function post(opts: PostOptions): SplitArticle[] { const random = rng(opts.seed ?? 1) const connections = opts.connections ?? 1 const label = opts.label ?? 'p' const queue: { volume: number; number: number }[] = [] for (let v = 0; v < opts.volumes; v++) for (let n = 1; n <= opts.parts; n++) queue.push({ volume: v, number: n }) const articles: SplitArticle[] = [] let articleNumber = opts.start ?? 1000 for (let i = 0; i < queue.length; i += connections) { // One round: each connection posts one article; they land in random order. const round = queue.slice(i, i + connections).sort(() => random() - 0.5) for (const { volume, number } of round) { articleNumber += 1 + Math.floor(random() * (opts.noise ?? 0)) if (random() < (opts.drop ?? 0)) continue articles.push({ number, messageId: `${label}${volume}.${number}`, bytes: 700_000, articleNumber }) } } return articles } /** Source volume of an article, from its simulated message-id. */ const source = (a: SplitArticle) => a.messageId.split('.')[0] /** Every result volume must contain articles of one source volume only. */ function assertPure(assert: any, volumes: SplitArticle[][]) { for (const volume of volumes) { assert.lengthOf(new Set(volume.map(source)), 1, `mixed volume: ${volume.map((a) => a.messageId).join(' ')}`) } } test.group('split / splitInstances', () => { test('separates sequentially posted volumes', ({ assert }) => { const volumes = splitInstances(post({ volumes: 3, parts: 14 }), { totalParts: 14 }) assert.lengthOf(volumes, 3) assertPure(assert, volumes) assert.isTrue(volumes.every((v) => v.length === 14)) }) test('separates volumes posted over interleaved connections, out of order', ({ assert }) => { for (const seed of [1, 2, 3, 4, 5]) { const volumes = splitInstances(post({ volumes: 6, parts: 40, connections: 8, noise: 30, seed }), { totalParts: 40 }) assertPure(assert, volumes) assert.lengthOf(volumes, 6, `seed ${seed}`) } }) test('a volume that completes is never mixed, even with heavy loss', ({ assert }) => { for (const seed of [11, 12, 13, 14, 15, 16, 17, 18]) { const articles = post({ volumes: 5, parts: 30, connections: 6, noise: 20, drop: 0.2, seed }) const volumes = splitInstances(articles, { totalParts: 30 }) assertPure(assert, volumes.filter((v) => v.length === 30)) } }) test('a missing article only affects its own volume', ({ assert }) => { const articles = post({ volumes: 4, parts: 20, connections: 4, drop: 0.05, seed: 7 }) const volumes = splitInstances(articles, { totalParts: 20 }) assertPure(assert, volumes) assert.lengthOf(volumes, 4) }) test('posts of the same size at different times do not mix', ({ assert }) => { // An earlier post that lost articles (e.g. before the backfill window), then two more. const articles = [ ...post({ volumes: 3, parts: 14, connections: 4, drop: 0.3, label: 'a', start: 1_000, seed: 3 }), ...post({ volumes: 3, parts: 14, connections: 4, label: 'b', start: 200_000, seed: 4 }), ...post({ volumes: 2, parts: 14, connections: 2, label: 'c', start: 900_000, seed: 5 }), ] const volumes = splitInstances(articles, { totalParts: 14 }) // The damaged post may produce mixed volumes, but those stay incomplete // (they never become files); every volume that completes is pure. const complete = volumes.filter((v) => v.length === 14) assertPure(assert, complete) assert.includeMembers(complete.map((v) => source(v[0])), ['b0', 'b1', 'b2', 'c0', 'c1']) // The later, intact posts are not affected at all. assertPure(assert, volumes.filter((v) => !source(v[0]).startsWith('a'))) }) test('ignores duplicate message-ids and does not depend on input order', ({ assert }) => { const articles = post({ volumes: 2, parts: 5, connections: 3, seed: 9 }) const shuffled = [...articles].reverse().concat(articles[0]) assert.deepEqual(splitInstances(shuffled, { totalParts: 5 }), splitInstances(articles, { totalParts: 5 })) }) }) /** Like post(), with real data per volume so part and file CRCs are genuine. */ function postWithCrcs(opts: PostOptions): SplitArticle[] { const random = rng((opts.seed ?? 1) + 1000) const size = 64 const data = new Map() const articles = post(opts) for (const a of articles) { const volume = source(a) if (!data.has(volume)) { data.set(volume, Buffer.from(Array.from({ length: opts.parts * size }, () => Math.floor(random() * 256)))) } const file = data.get(volume)! a.pcrc32 = crc32Hex(file.subarray((a.number - 1) * size, a.number * size)) a.partSize = size a.fileCrc32 = a.number === opts.parts ? crc32Hex(file) : null } return articles } test.group('split / verified', () => { test('verifyVolume accepts a correct volume and rejects a mixed one', ({ assert }) => { const articles = postWithCrcs({ volumes: 2, parts: 6 }) const volumeA = articles.filter((a) => source(a) === 'p0') assert.isTrue(verifyVolume(volumeA, 6)) const mixed = volumeA.map((a) => (a.number === 3 ? articles.find((b) => source(b) === 'p1' && b.number === 3)! : a)) assert.isFalse(verifyVolume(mixed, 6)) assert.isFalse(verifyVolume(volumeA.slice(0, 5), 6)) }) test('only verified volumes come out, and they are never mixed', ({ assert }) => { for (const seed of [21, 22, 23, 24, 25, 26]) { const articles = [ ...postWithCrcs({ volumes: 4, parts: 30, connections: 16, noise: 20, drop: 0.05, label: 'a', seed }), ...postWithCrcs({ volumes: 3, parts: 30, connections: 16, noise: 20, label: 'b', start: 300_000, seed: seed + 50 }), ] for (const volume of splitVerified(articles, { totalParts: 30 })) { assert.isTrue(verifyVolume(volume, 30)) assertPure(assert, [volume]) } } }) test('recovers every volume of a cleanly posted set', ({ assert }) => { const volumes = splitVerified(postWithCrcs({ volumes: 5, parts: 20, connections: 4, noise: 10, seed: 31 }), { totalParts: 20 }) assert.lengthOf(volumes, 5) }) })