import { test } from '@japa/runner'
import { RangeIndexer } from '#lib/edge/index'
import { decodeChunk } from '#lib/chunks/index'
import { DEFAULT_RULES } from '#lib/rules/index'
import type { Overview } from '#lib/nntp/index'
const GROUP = 'alt.binaries.test'
const rules = { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { 'alt.binaries.boneless': 1, [GROUP]: 2, 'alt.binaries.x': 3 } }
let id = 0
/** Overview lines of one posted file: all its parts, from article number `at` on. */
function post(at: number, name: string, parts: number, opts: { counter?: string; groups?: string[]; tool?: 'ngPost' | 'nyuu'; poster?: string } = {}): Overview[] {
return Array.from({ length: parts }, (_, i) => {
id++
const mid = opts.tool === 'ngPost' ? `${id.toString(16).padStart(32, '0')}@ngPost` : `art${id}@test`
const groups = opts.groups ?? [GROUP]
return [at + i, {
'subject': `${opts.counter ? `${opts.counter} - ` : ''}"${name}" yEnc (${i + 1}/${parts})`,
'from': opts.poster ?? 'poster
',
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
'message-id': `<${mid}>`,
':bytes': '700000',
'xref': `news.example ${groups.map((g, k) => `${g}:${at + i + k}`).join(' ')}`,
}] as Overview
})
}
test.group('edge / RangeIndexer', () => {
test('groups files into releases with probe candidates, and counts what the rules dropped', ({ assert }) => {
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
indexer.add([
...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 3, { counter: 'Some.Show.S01E01 [1/3]' }),
...post(100_010, 'Some.Show.S01E01.1080p.WEB.part2.rar', 3, { counter: 'Some.Show.S01E01 [2/3]' }),
...post(100_020, 'Some.Show.S01E01.1080p.WEB.par2', 1, { counter: 'Some.Show.S01E01 [3/3]' }),
// A crosspost whose owner is boneless (priority 1): skipped here.
...post(100_030, 'Other.Release.2020.mkv', 2, { groups: [GROUP, 'alt.binaries.boneless'] }),
// Random-named ngPost upload: garbage everywhere.
...post(100_040, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 4, { tool: 'ngPost' }),
// A random subject per article: per-article.
...post(100_050, 'Pq4ZmXc8VbN2LkJh6GfDs3', 1),
], 200_000)
indexer.add([[100_060, { 'subject': 'obfuscated', 'from': 'x', 'message-id': '', 'date': '', ':bytes': '1', 'xref': `news.example ${GROUP}:100060` }]], 300_000)
indexer.finish()
const batch = indexer.take()
assert.equal(batch.headers, 3 + 3 + 1 + 2 + 4 + 1 + 1)
assert.lengthOf(batch.releases, 1)
const [release] = batch.releases
assert.include(release, { files: 3, segments: 7, firstArticle: 100_000, lastArticle: 100_020 })
assert.deepEqual(decodeChunk(release.payload).map((f) => f.name).sort(), ['Some.Show.S01E01.1080p.WEB.par2', 'Some.Show.S01E01.1080p.WEB.part1.rar', 'Some.Show.S01E01.1080p.WEB.part2.rar'])
assert.deepEqual(batch.candidates.map((c) => [c.kind, c.file]).sort(), [['archive', 'Some.Show.S01E01.1080p.WEB.part1.rar'], ['par2', 'Some.Show.S01E01.1080p.WEB.par2']])
const counts = Object.fromEntries(batch.counts.map((c) => [`${c.rule} ${c.key}`.trim(), [c.articles, c.files]]))
assert.deepEqual(counts, { 'crosspost alt.binaries.boneless': [2, 0], 'garbage ngPost alt.binaries.test': [4, 1], 'per-article other': [1, 1], 'obfuscated': [1, 0] })
assert.equal(batch.next, 1_000_001)
})
test("archive candidates: a split 7z's start and its highest volume's end, an unsplit 7z's start and end, a ZIP's start", ({ assert }) => {
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
indexer.add([
...post(100, 'Split.Seven.7z.001', 2, { counter: 'Split.Seven [1/3]' }),
...post(110, 'Split.Seven.7z.003', 2, { counter: 'Split.Seven [3/3]' }),
...post(120, 'Split.Seven.7z.002', 2, { counter: 'Split.Seven [2/3]' }),
...post(200, 'Whole.Seven.7z', 3, { counter: 'Whole.Seven [1/1]' }),
...post(300, 'Some.Zip.zip', 2, { counter: 'Some.Zip [1/1]' }),
], 200_000)
indexer.finish()
const got = indexer.take().candidates.map((c) => `${c.kind} ${c.file}`).sort()
assert.deepEqual(got, ['7z-tail Split.Seven.7z.003', '7z-tail Whole.Seven.7z', 'archive Some.Zip.zip', 'archive Split.Seven.7z.001', 'archive Whole.Seven.7z'])
})
test('files the edges cut leave as fragments; one missing parts in the middle is only counted', ({ assert }) => {
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
indexer.add([
// Parts 2–3 right after the start: part 1 is in the previous range.
...post(5, 'Start.Cut.2020.part1.rar', 3).slice(1),
// Parts 1–2 seen, part 3 never: incomplete on the server.
...post(500_000, 'Middle.Hole.2020.part1.rar', 3).slice(0, 2),
], 600_000)
// Chunks without its part 3: the hole file is given up (idle) long before the end.
for (let at = 650_000; at < 950_000; at += 50_000) indexer.add([], at)
// Parts 1–2 near the end, still open when the range ends: part 3 is in the next range.
indexer.add(post(999_990, 'End.Cut.2020.part1.rar', 3).slice(0, 2), 1_000_001)
indexer.finish()
const batch = indexer.take()
assert.deepEqual(batch.fragments!.map((f) => [f.name, f.segments.map((s) => s[0])]).sort(), [['End.Cut.2020.part1.rar', [1, 2]], ['Start.Cut.2020.part1.rar', [2, 3]]])
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]])
assert.lengthOf(batch.releases, 0)
})
})
test.group('edge / taking over a range', () => {
test("a worker that resumes a range midway starts its first batch at the range's resume point", ({ assert }) => {
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, resumeAt: 500_001, rules })
indexer.add(post(500_001, 'A.Release.2020.mkv', 1), 600_000)
assert.deepEqual([indexer.take().from, indexer.take().from], [500_001, 600_000])
})
test('uploads split where posting pauses for more than 3 hours, counted from the latest article', ({ assert }) => {
const art = (n: number, subject: string, minutes: number): Overview => [n, { 'subject': subject, 'from': 'p ', 'date': new Date(Date.UTC(2026, 9, 5, 10) + minutes * 60_000).toUTCString(), 'message-id': ``, ':bytes': '1000', 'xref': `news.example ${GROUP}:${n}` }]
const ix = new RangeIndexer({ group: GROUP, start: 1, end: 10_000, rules })
ix.add([
// A big file posted over 4 hours, then its par2 right after: one upload.
...Array.from({ length: 5 }, (_, i) => art(100 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, i * 60)),
art(200, 'Big.Release [2/2] - "Big.Release.par2" yEnc (1/1)', 245),
// The same file again a day later: another.
...Array.from({ length: 5 }, (_, i) => art(300 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, 1440 + i)),
], 10_001)
ix.finish()
const batch = ix.take()
assert.deepEqual(batch.releases.map((r) => [r.upload, r.files]), [[0, 2], [1, 1]])
assert.equal(batch.releases[0].postedLast! - batch.releases[0].postedAt!, 245 * 60_000)
assert.equal(batch.releases[0].key, batch.releases[1].key)
})
})