usenet-indexer/tests/unit/edge/range_indexer.spec.ts

import { test } from '@japa/runner'
import { RangeIndexer } from '#lib/edge/index'
import { decodeChunk } from '#lib/chunks/index'
import { DEFAULT_RULES } from '#lib/rules/index'
import type { Overview } from '#lib/nntp/index'

const GROUP = 'alt.binaries.test'
const rules = { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { 'alt.binaries.boneless': 1, [GROUP]: 2, 'alt.binaries.x': 3 } }
let id = 0

/** Overview lines of one posted file: all its parts, from article number `at` on. */
function post(at: number, name: string, parts: number, opts: { counter?: string; groups?: string[]; tool?: 'ngPost' | 'nyuu'; poster?: string } = {}): Overview[] {
  return Array.from({ length: parts }, (_, i) => {
    id++
    const mid = opts.tool === 'ngPost' ? `${id.toString(16).padStart(32, '0')}@ngPost` : `art${id}@test`
    const groups = opts.groups ?? [GROUP]
    return [at + i, {
      'subject': `${opts.counter ? `${opts.counter} - ` : ''}"${name}" yEnc (${i + 1}/${parts})`,
      'from': opts.poster ?? 'poster <p@x>',
      'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
      'message-id': `<${mid}>`,
      ':bytes': '700000',
      'xref': `news.example ${groups.map((g, k) => `${g}:${at + i + k}`).join(' ')}`,
    }] as Overview
  })
}

test.group('edge / RangeIndexer', () => {
  test('groups files into releases with probe candidates, and counts what the rules dropped', ({ assert }) => {
    const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
    indexer.add([
      ...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 3, { counter: 'Some.Show.S01E01 [1/3]' }),
      ...post(100_010, 'Some.Show.S01E01.1080p.WEB.part2.rar', 3, { counter: 'Some.Show.S01E01 [2/3]' }),
      ...post(100_020, 'Some.Show.S01E01.1080p.WEB.par2', 1, { counter: 'Some.Show.S01E01 [3/3]' }),
      // A crosspost whose owner is boneless (priority 1): skipped here.
      ...post(100_030, 'Other.Release.2020.mkv', 2, { groups: [GROUP, 'alt.binaries.boneless'] }),
      // Random-named ngPost upload: garbage everywhere.
      ...post(100_040, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 4, { tool: 'ngPost' }),
      // A random subject per article: per-article.
      ...post(100_050, 'Pq4ZmXc8VbN2LkJh6GfDs3', 1),
    ], 200_000)
    indexer.add([[100_060, { 'subject': 'obfuscated', 'from': 'x', 'message-id': '<obf@test>', 'date': '', ':bytes': '1', 'xref': `news.example ${GROUP}:100060` }]], 300_000)
    indexer.finish()
    const batch = indexer.take()

    assert.equal(batch.headers, 3 + 3 + 1 + 2 + 4 + 1 + 1)
    assert.lengthOf(batch.releases, 1)
    const [release] = batch.releases
    assert.include(release, { files: 3, segments: 7, firstArticle: 100_000, lastArticle: 100_020 })
    assert.deepEqual(decodeChunk(release.payload).map((f) => f.name).sort(), ['Some.Show.S01E01.1080p.WEB.par2', 'Some.Show.S01E01.1080p.WEB.part1.rar', 'Some.Show.S01E01.1080p.WEB.part2.rar'])
    assert.deepEqual(batch.candidates.map((c) => [c.kind, c.file]).sort(), [['archive', 'Some.Show.S01E01.1080p.WEB.part1.rar'], ['par2', 'Some.Show.S01E01.1080p.WEB.par2']])

    const counts = Object.fromEntries(batch.counts.map((c) => [`${c.rule} ${c.key}`.trim(), [c.articles, c.files]]))
    assert.deepEqual(counts, { 'crosspost alt.binaries.boneless': [2, 0], 'garbage ngPost alt.binaries.test': [4, 1], 'per-article other': [1, 1], 'obfuscated': [1, 0] })
    assert.equal(batch.next, 1_000_001)
  })

  test("archive candidates: a split 7z's start and its highest volume's end, an unsplit 7z's start and end, a ZIP's start", ({ assert }) => {
    const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
    indexer.add([
      ...post(100, 'Split.Seven.7z.001', 2, { counter: 'Split.Seven [1/3]' }),
      ...post(110, 'Split.Seven.7z.003', 2, { counter: 'Split.Seven [3/3]' }),
      ...post(120, 'Split.Seven.7z.002', 2, { counter: 'Split.Seven [2/3]' }),
      ...post(200, 'Whole.Seven.7z', 3, { counter: 'Whole.Seven [1/1]' }),
      ...post(300, 'Some.Zip.zip', 2, { counter: 'Some.Zip [1/1]' }),
    ], 200_000)
    indexer.finish()
    const got = indexer.take().candidates.map((c) => `${c.kind} ${c.file}`).sort()
    assert.deepEqual(got, ['7z-tail Split.Seven.7z.003', '7z-tail Whole.Seven.7z', 'archive Some.Zip.zip', 'archive Split.Seven.7z.001', 'archive Whole.Seven.7z'])
  })

  test('files the edges cut leave as fragments; one missing parts in the middle is only counted', ({ assert }) => {
    const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
    indexer.add([
      // Parts 2–3 right after the start: part 1 is in the previous range.
      ...post(5, 'Start.Cut.2020.part1.rar', 3).slice(1),
      // Parts 1–2 seen, part 3 never: incomplete on the server.
      ...post(500_000, 'Middle.Hole.2020.part1.rar', 3).slice(0, 2),
    ], 600_000)
    // Chunks without its part 3: the hole file is given up (idle) long before the end.
    for (let at = 650_000; at < 950_000; at += 50_000) indexer.add([], at)
    // Parts 1–2 near the end, still open when the range ends: part 3 is in the next range.
    indexer.add(post(999_990, 'End.Cut.2020.part1.rar', 3).slice(0, 2), 1_000_001)
    indexer.finish()
    const batch = indexer.take()
    assert.deepEqual(batch.fragments!.map((f) => [f.name, f.segments.map((s) => s[0])]).sort(), [['End.Cut.2020.part1.rar', [1, 2]], ['Start.Cut.2020.part1.rar', [2, 3]]])
    assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]])
    assert.lengthOf(batch.releases, 0)
  })

  test("partialFiles: a file short of parts is a partial release, as the Rust window writes it; one article of a random multi-part file isn't", ({ assert }) => {
    const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 200, rules, idleChunks: 0, edgeArticles: 0, partialFiles: true })
    const name = '8c21f5c36450f9e2ccc6ff387d159d4d'
    indexer.add([
      // Parts 1, 3 and 5 of 6 (the others gone from the server), from a re-send's copy of part 3 too.
      ...post(100, name, 6, { poster: 'Bob <bob@home.me>' }).filter((_, i) => i % 2 === 0),
      ...post(150, name, 6, { poster: 'Bob <bob@home.me>' }).slice(2, 3),
      ...post(160, 'Pq4ZmXc8VbN2LkJh6GfDs3', 4).slice(1, 2),
    ], 201)
    indexer.finish()
    const batch = indexer.take()
    assert.lengthOf(batch.releases, 1)
    assert.include(batch.releases[0], { name, files: 1, segments: 3, partial: true, firstArticle: 100, lastArticle: 104 })
    assert.deepEqual(decodeChunk(batch.releases[0].payload)[0].segments.map((s) => s[0]), [1, 3, 5])
    assert.deepEqual(batch.fragments, [])
    assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]])
  })

  test('partialFiles: one article of a multi-part file under a scrambled name is per-article before the end, a partial release at it', ({ assert }) => {
    const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 1_000_000, rules, idleChunks: 1, edgeArticles: 0, partialFiles: true })
    // Part 8 of 67, a random poster's: given up mid-range.
    indexer.add(post(100, 'nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr', 67, { poster: 'guAym5TqSlp SCTLKQ0R <8nGc3Rk@OrjrYqFkp.z>' }).slice(7, 8), 200)
    indexer.add([], 300)
    indexer.add([], 400)
    // The same shape still open when the block ends: the rest can be in the next one.
    indexer.add(post(999_990, 'bslXlwgPiqZadhRa', 5).slice(1, 2), 1_000_001)
    indexer.finish()
    const batch = indexer.take()
    assert.deepEqual(batch.releases.map((r) => [r.name, r.partial]), [['bslXlwgPiqZadhRa', true]])
    assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]])
  })
})

test.group('edge / taking over a range', () => {
  test("a worker that resumes a range midway starts its first batch at the range's resume point", ({ assert }) => {
    const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, resumeAt: 500_001, rules })
    indexer.add(post(500_001, 'A.Release.2020.mkv', 1), 600_000)
    assert.deepEqual([indexer.take().from, indexer.take().from], [500_001, 600_000])
  })
  test('uploads split where posting pauses for more than 3 hours, counted from the latest article', ({ assert }) => {
    const art = (n: number, subject: string, minutes: number): Overview => [n, { 'subject': subject, 'from': 'p <p@x>', 'date': new Date(Date.UTC(2026, 9, 5, 10) + minutes * 60_000).toUTCString(), 'message-id': `<u${n}@test>`, ':bytes': '1000', 'xref': `news.example ${GROUP}:${n}` }]
    const ix = new RangeIndexer({ group: GROUP, start: 1, end: 10_000, rules })
    ix.add([
      // A big file posted over 4 hours, then its par2 right after: one upload.
      ...Array.from({ length: 5 }, (_, i) => art(100 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, i * 60)),
      art(200, 'Big.Release [2/2] - "Big.Release.par2" yEnc (1/1)', 245),
      // The same file again a day later: another.
      ...Array.from({ length: 5 }, (_, i) => art(300 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, 1440 + i)),
    ], 10_001)
    ix.finish()
    const batch = ix.take()
    assert.deepEqual(batch.releases.map((r) => [r.upload, r.files]), [[0, 2], [1, 1]])
    assert.equal(batch.releases[0].postedLast! - batch.releases[0].postedAt!, 245 * 60_000)
    assert.equal(batch.releases[0].key, batch.releases[1].key)
  })
})