usenet-indexer/tests/unit/probe.spec.ts

import { test } from '@japa/runner'
import { crc32Hex } from '#lib/yenc/index'
import { probeInspection, type InspectTask } from '#lib/content/probe'

/** One yEnc article of `file`: part `index` of `size` bytes (escaping '.' and whitespace too, so no dot-stuffing). */
function yencArticle(name: string, file: Buffer, index: number, size: number) {
  const data = file.subarray(index * size, (index + 1) * size)
  const total = Math.ceil(file.length / size)
  const out: number[] = []
  for (const byte of data) {
    const c = (byte + 42) & 0xff
    if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
    else out.push(c)
  }
  const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
  const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
  return Buffer.from(head + Buffer.from(out).toString('latin1') + tail, 'latin1')
}

/** A posted file as articles, and its inspection task. */
function posted(name: string, file: Buffer, size = 400) {
  const articles = new Map<string, Buffer>()
  const segments = []
  for (let i = 0; i * size < file.length; i++) {
    const id = `${name}-${i}@test`
    articles.set(id, yencArticle(name, file, i, size))
    segments.push({ number: i + 1, id, size: size + 100 })
  }
  const fetched: string[] = []
  const fetch = async (id: string) => {
    fetched.push(id)
    const body = articles.get(id)
    if (!body) throw new Error(`430 no such article ${id}`)
    return body
  }
  const task: InspectTask = { kind: 'inspect', fileId: 1, name, segments, sevenZip: { volumes: 1, last: segments } }
  return { task, fetch, fetched }
}

// `7z a -pSecret -mhe=on`: an encrypted header (tests/unit/sevenzip.spec.ts).
const HEADER_ENCRYPTED = Buffer.from(
  'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
  'base64'
)

test.group('content probe', () => {
  test('a first segment gives size, fingerprint and the yEnc name', async ({ assert }) => {
    const file = Buffer.alloc(20_000, 7)
    const { task, fetch, fetched } = posted('Some.Name.bin', file, 17_000)
    const probe = await probeInspection(task, fetch)
    assert.include(probe, { size: 20_000, contentType: null, yencName: 'Some.Name.bin' })
    assert.match(probe.md5_16k!, /^[0-9a-f]{32}$/)
    assert.deepEqual(fetched, ['Some.Name.bin-0@test'])
  })

  test('an NZB is read whole', async ({ assert }) => {
    const xml = `<?xml version="1.0"?>\n<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${'<file subject="x"/>'.repeat(60)}</nzb>`
    const { task, fetch, fetched } = posted('x.nzb', Buffer.from(xml))
    const probe = await probeInspection(task, fetch)
    assert.equal(probe.contentType, 'nzb')
    assert.equal(probe.text, xml)
    // Every segment once: the first, decoded to sniff, is reused for the whole.
    assert.deepEqual(fetched, task.segments.map((s) => s.id))
  })

  test('a 7z archive with encrypted headers is known from its tail, without bsdtar', async ({ assert }) => {
    const { task, fetch } = posted('a.7z', HEADER_ENCRYPTED, 100)
    const probe = await probeInspection(task, fetch)
    assert.equal(probe.contentType, '7z')
    assert.include(probe.archive!, { format: '7z', headersEncrypted: true })
    assert.lengthOf(probe.archive!.entries, 0)
  })

  test('a missing first article is a result, not a failure (no retries)', async ({ assert }) => {
    const { task, fetch } = posted('gone.bin', Buffer.alloc(10, 1))
    task.segments[0].id = 'missing@test'
    assert.include(await probeInspection(task, fetch), { contentType: 'missing' })
    // Through the seed relay: 410.
    const relayed = await probeInspection(task, async () => {
      throw new Error('Seed API /api/v1/seed/articles?id=x: 410 {"error":"No such article"}')
    })
    assert.equal(relayed.contentType, 'missing')
    // Anything else still fails (and is retried).
    await assert.rejects(() => probeInspection(task, async () => { throw new Error('NNTP BODY: no answer within 120 s') }), /no answer/)
  })
})