usenet-indexer/tests/unit/probe/inspect.spec.ts

import { test } from '@japa/runner'
import { createHash } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { crc32 } from 'node:zlib'
import yencode from 'yencode'
import { crc32Hex } from '#lib/yenc/index'
import { inspect, innerName, MAX_NAMES } from '#lib/probe/index'

/** A single-part yEnc article of `data`. */
function article(data: Buffer, name: string) {
  const lines = [`=ybegin line=128 size=${data.length} name=${name}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} crc32=${crc32Hex(data)}`]
  return Buffer.from(lines.join('\r\n'), 'latin1')
}

const md5 = (data: Buffer) => createHash('md5').update(data).digest()
/** Builds one PAR2 packet with a valid checksum. */
function packet(setId: Buffer, type: string, body: Buffer) {
  const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
  const length = Buffer.alloc(8)
  length.writeBigUInt64LE(BigInt(64 + padded.length))
  const typeBuf = Buffer.alloc(16)
  typeBuf.write(type, 'latin1')
  const hashed = Buffer.concat([setId, typeBuf, padded])
  return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
}
function fileDesc(setId: Buffer, name: string, size: number) {
  const len = Buffer.alloc(8)
  len.writeBigUInt64LE(BigInt(size))
  const md5_16k = md5(Buffer.from(`${name}-16k`))
  const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
  const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
  return packet(setId, 'PAR 2.0\0FileDesc', body)
}
const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
function vint(value: number) {
  const bytes: number[] = []
  do {
    let byte = value % 128
    value = Math.floor(value / 128)
    if (value > 0) byte |= 0x80
    bytes.push(byte)
  } while (value > 0)
  return Buffer.from(bytes)
}
function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
  let flags = headerFlags
  const parts = [vint(type)]
  const sizes: Buffer[] = []
  if (opts.extra) {
    flags |= 0x0001
    sizes.push(vint(opts.extra.length))
  }
  if (opts.data !== undefined) {
    flags |= 0x0002
    sizes.push(vint(opts.data))
  }
  const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
  const sized = Buffer.concat([vint(header.length), header])
  const crc = Buffer.alloc(4)
  crc.writeUInt32LE(crc32(sized) >>> 0)
  return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
}
function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
  const nameBuf = Buffer.from(name, 'utf8')
  const fields = Buffer.concat([
    vint(opts.dir ? 0x0001 : 0), // file flags
    vint(size),
    vint(0x20), // attributes
    vint(0), // compression info
    vint(0), // host OS
    vint(nameBuf.length),
    nameBuf,
  ])
  // Encryption record: size, type 1, then its data (dummy here).
  const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
  return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
}

test.group('probe / inspect', () => {
  test('a par2 index: its files, largest first', ({ assert }) => {
    const setId = Buffer.alloc(16, 3)
    const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)])
    const result = inspect(article(par2, 'a8f3k2.par2'))
    const { par2: set, hash16k, ...rest } = result
    assert.deepEqual(rest, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
    // The whole set for the par2 merge: per file the hash of its first 16 KiB, its length and name.
    assert.equal(set!.setId, setId.toString('hex'))
    assert.deepEqual(
      set!.files.map((f) => [f.name, f.length, f.hash16k]),
      [
        ['Movie.2020.part02.rar', 100, md5(Buffer.from('Movie.2020.part02.rar-16k')).toString('hex')],
        ['Movie.2020.part01.rar', 200, md5(Buffer.from('Movie.2020.part01.rar-16k')).toString('hex')],
      ]
    )
    assert.equal(hash16k, md5(par2).toString('hex'))
  })

  test("the hash of a file's first 16 KiB, when the article starts the file", ({ assert }) => {
    const data = Buffer.alloc(40_000, 7)
    assert.equal(inspect(article(data, 'x')).hash16k, md5(data.subarray(0, 16384)).toString('hex'))
    const part = Buffer.from(
      [`=ybegin part=2 total=3 line=128 size=120000 name=x`, `=ypart begin=40001 end=80000`, yencode.encode(data).toString('latin1'), `=yend size=40000 part=2 pcrc32=${crc32Hex(data)}`].join('\r\n'),
      'latin1'
    )
    assert.isUndefined(inspect(part).hash16k)
  })

  test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => {
    const rar = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('Movie.2020.1080p/movie.mkv', 5_000_000, { packed: 10 }), rar5File('Movie.2020.1080p/sample.mkv', 1000, { packed: 10 })])
    const open = inspect(article(rar, 'x7q.part01.rar'))
    assert.include(open, { type: 'rar5', entries: 2, passworded: false })
    assert.deepEqual(open.names, ['Movie.2020.1080p/movie.mkv', 'Movie.2020.1080p/sample.mkv'])
    const locked = inspect(article(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]), 'y.rar'))
    assert.isTrue(locked.passworded)
    const hidden = inspect(article(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]), 'z.rar'))
    assert.include(hidden, { passworded: true, entries: 0 })
  })

  test('other data: the yEnc name and what the bytes are', ({ assert }) => {
    const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)])
    assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska', hash16k: md5(mkv).toString('hex') })
  })

  test('not yEnc: throws', ({ assert }) => {
    assert.throws(() => inspect(Buffer.from('just some text\r\n')))
  })

  test('at most MAX_NAMES names', ({ assert }) => {
    const setId = Buffer.alloc(16, 4)
    const par2 = Buffer.concat(Array.from({ length: MAX_NAMES + 5 }, (_, i) => fileDesc(setId, `f${i}.rar`, i)))
    const result = inspect(article(par2, 'p.par2'))
    assert.equal(result.entries, MAX_NAMES + 5)
    assert.lengthOf(result.names!, MAX_NAMES)
  })
})

test.group('probe / innerName', () => {
  test("the archive's largest entry, else par2's largest non-packaging file, else a lone file's yEnc name", ({ assert }) => {
    const archive = { yencName: 'a.rar', size: 1, type: 'rar5' as const, names: ['Show.S01E01/show.s01e01.mkv'] }
    const par2 = { yencName: 'a.par2', size: 1, type: 'par2' as const, names: ['abc.part01.rar', 'Show.S01E01.1080p.mkv'] }
    assert.equal(innerName([{ kind: 'par2', result: par2 }, { kind: 'archive', result: archive }]), 'show.s01e01.mkv')
    assert.equal(innerName([{ kind: 'par2', result: par2 }]), 'Show.S01E01.1080p.mkv')
    assert.equal(innerName([{ kind: 'par2', result: { ...par2, names: ['abc.part01.rar', 'abc.par2'] } }]), null)
    const lone = { kind: 'lone', result: { yencName: 'Real.Name.mkv', size: 1, type: 'matroska' as const } }
    assert.equal(innerName([lone], 'k3j4h5'), 'Real.Name.mkv')
    // A random-named file whose yEnc name is its random name says nothing new.
    assert.isNull(innerName([lone], 'real.name.mkv'))
    assert.isNull(innerName([lone], 'Real.Name'))
    // Nor does packaging: a lone par2 volume.
    assert.isNull(innerName([{ kind: 'lone', result: { yencName: 'a1b2.vol007+08.par2', size: 1, type: 'par2' } }], 'x9y8'))
  })

  test("a 7z-tail: the listing from the last segment of the last volume; one that isn't the last segment: none", ({ assert }) => {
    const tail = readFileSync(new URL('../../fixtures/sevenzip/split-last-volume.tail', import.meta.url))
    // The volume's last 8,000 bytes as its third part, of a volume 1,000,000 bytes long.
    const last = tail.subarray(tail.length - 8000)
    const part = (data: Buffer, begin: number) =>
      Buffer.from(
        [`=ybegin part=3 total=3 line=128 size=1000000 name=x.7z.004`, `=ypart begin=${begin} end=${begin + data.length - 1}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} part=3 pcrc32=${crc32Hex(data)}`].join('\r\n'),
        'latin1'
      )
    const result = inspect(part(last, 1_000_000 - 8000 + 1), '7z-tail')
    assert.equal(result.entries, 4)
    assert.deepEqual(result.names?.[0], 'Some.Movie.2020.1080p.BluRay.x264-GRP/some.movie.2020.1080p.bluray.x264-grp.mkv')
    assert.isFalse(result.passworded)
    assert.isUndefined(inspect(part(last, 1), '7z-tail').names)
    const encrypted = readFileSync(new URL('../../fixtures/sevenzip/header-encrypted.tail', import.meta.url))
    const hidden = inspect(part(encrypted.subarray(encrypted.length - 8000), 1_000_000 - 8000 + 1), '7z-tail')
    assert.isTrue(hidden.passworded)
    assert.equal(hidden.entries, 0)
    assert.isUndefined(hidden.names)
    // A small 7z whole in one article, probed as an archive start.
    assert.deepEqual(inspect(article(readFileSync(new URL('../../fixtures/sevenzip/tiny.7z', import.meta.url)), 'x.7z'), 'archive').names, ['b.nfo', 'a.txt'])
  })

  test('a ZIP start: the names of its first entries, and whether they are encrypted', ({ assert }) => {
    const local = (name: string, data: Buffer, flags = 0) => {
      const header = Buffer.alloc(30)
      header.writeUInt32LE(0x04034b50, 0)
      header.writeUInt16LE(flags, 6)
      header.writeUInt32LE(data.length, 18)
      header.writeUInt32LE(data.length, 22)
      header.writeUInt16LE(Buffer.byteLength(name), 26)
      return Buffer.concat([header, Buffer.from(name), data])
    }
    const zip = Buffer.concat([local('Album/', Buffer.alloc(0)), local('Album/01 - Track.flac', Buffer.alloc(3000, 1)), local('Album/cover.jpg', Buffer.alloc(500, 2))])
    const plain = inspect(article(zip, 'album.zip'))
    assert.equal(plain.type, 'zip')
    assert.deepEqual(plain.names, ['Album/01 - Track.flac', 'Album/cover.jpg'])
    assert.isFalse(plain.passworded)
    assert.isTrue(inspect(article(local('secret.mkv', Buffer.alloc(100), 0x1), 'x.zip')).passworded)
  })
})