import { test } from '@japa/runner' import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' import { crc32 } from 'node:zlib' import yencode from 'yencode' import { crc32Hex } from '#lib/yenc/index' import { inspect, innerName, MAX_NAMES, usableName } from '#lib/probe/index' /** A single-part yEnc article of `data`. */ function article(data: Buffer, name: string) { const lines = [`=ybegin line=128 size=${data.length} name=${name}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} crc32=${crc32Hex(data)}`] return Buffer.from(lines.join('\r\n'), 'latin1') } const md5 = (data: Buffer) => createHash('md5').update(data).digest() /** Builds one PAR2 packet with a valid checksum. */ function packet(setId: Buffer, type: string, body: Buffer) { const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)]) const length = Buffer.alloc(8) length.writeBigUInt64LE(BigInt(64 + padded.length)) const typeBuf = Buffer.alloc(16) typeBuf.write(type, 'latin1') const hashed = Buffer.concat([setId, typeBuf, padded]) return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed]) } function fileDesc(setId: Buffer, name: string, size: number) { const len = Buffer.alloc(8) len.writeBigUInt64LE(BigInt(size)) const md5_16k = md5(Buffer.from(`${name}-16k`)) const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)])) const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')]) return packet(setId, 'PAR 2.0\0FileDesc', body) } const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]) function vint(value: number) { const bytes: number[] = [] do { let byte = value % 128 value = Math.floor(value / 128) if (value > 0) byte |= 0x80 bytes.push(byte) } while (value > 0) return Buffer.from(bytes) } function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) { let flags = headerFlags const parts = [vint(type)] const sizes: Buffer[] = [] if (opts.extra) { flags |= 0x0001 sizes.push(vint(opts.extra.length)) } if (opts.data !== undefined) { flags |= 0x0002 sizes.push(vint(opts.data)) } const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)]) const sized = Buffer.concat([vint(header.length), header]) const crc = Buffer.alloc(4) crc.writeUInt32LE(crc32(sized) >>> 0) return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)]) } function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) { const nameBuf = Buffer.from(name, 'utf8') const fields = Buffer.concat([ vint(opts.dir ? 0x0001 : 0), // file flags vint(size), vint(0x20), // attributes vint(0), // compression info vint(0), // host OS vint(nameBuf.length), nameBuf, ]) // Encryption record: size, type 1, then its data (dummy here). const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed }) } test.group('probe / usable name (the pending ladder)', () => { test('a par2 or listing that names a real file names the release; random volumes and packaging do not', ({ assert }) => { const base = { size: 1000, type: null } assert.equal(usableName({ ...base, yencName: 'x.par2', names: ['q3B9OrpKiCxUnzgtUGnp.part01.rar', 'Some.Movie.2016.mkv'] }, 'par2'), 'Some.Movie.2016.mkv') assert.isNull(usableName({ ...base, yencName: 'q3B9OrpKiCxUnzgtUGnp.par2', names: ['q3B9OrpKiCxUnzgtUGnp.part01.rar'] }, 'par2')) assert.equal(usableName({ ...base, yencName: 'x.rar', names: ['Folder/Real.Name.S01E01.mkv'] }, 'archive'), 'Real.Name.S01E01.mkv') assert.isNull(usableName({ ...base, yencName: 'x.rar', names: [], passworded: true }, 'archive')) assert.equal(usableName({ ...base, yencName: 'brothalovers.20.02.16.hazel.moore.mp4' }, 'lone'), 'brothalovers.20.02.16.hazel.moore.mp4') assert.isNull(usableName({ ...base, yencName: 'c3453cdd2a3645a8f3cf3d22319abe63' }, 'lone')) assert.isNull(usableName({ ...base, yencName: 'c3453cdd2a3645a8f3cf3d22319abe63.part01.rar' }, 'lone')) }) }) test.group('probe / inspect', () => { test('a par2 index: its files, largest first', ({ assert }) => { const setId = Buffer.alloc(16, 3) const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)]) const result = inspect(article(par2, 'a8f3k2.par2')) const { par2: set, hash16k, ...rest } = result assert.deepEqual(rest, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] }) // The whole set for the par2 merge: per file the hash of its first 16 KiB, its length and name. assert.equal(set!.setId, setId.toString('hex')) assert.deepEqual( set!.files.map((f) => [f.name, f.length, f.hash16k]), [ ['Movie.2020.part02.rar', 100, md5(Buffer.from('Movie.2020.part02.rar-16k')).toString('hex')], ['Movie.2020.part01.rar', 200, md5(Buffer.from('Movie.2020.part01.rar-16k')).toString('hex')], ] ) assert.equal(hash16k, md5(par2).toString('hex')) }) test("the hash of a file's first 16 KiB, when the article starts the file", ({ assert }) => { const data = Buffer.alloc(40_000, 7) assert.equal(inspect(article(data, 'x')).hash16k, md5(data.subarray(0, 16384)).toString('hex')) const part = Buffer.from( [`=ybegin part=2 total=3 line=128 size=120000 name=x`, `=ypart begin=40001 end=80000`, yencode.encode(data).toString('latin1'), `=yend size=40000 part=2 pcrc32=${crc32Hex(data)}`].join('\r\n'), 'latin1' ) assert.isUndefined(inspect(part).hash16k) }) test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => { const rar = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('Movie.2020.1080p/movie.mkv', 5_000_000, { packed: 10 }), rar5File('Movie.2020.1080p/sample.mkv', 1000, { packed: 10 })]) const open = inspect(article(rar, 'x7q.part01.rar')) assert.include(open, { type: 'rar5', entries: 2, passworded: false }) assert.deepEqual(open.names, ['Movie.2020.1080p/movie.mkv', 'Movie.2020.1080p/sample.mkv']) const locked = inspect(article(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]), 'y.rar')) assert.isTrue(locked.passworded) const hidden = inspect(article(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]), 'z.rar')) assert.include(hidden, { passworded: true, entries: 0 }) }) test('other data: the yEnc name and what the bytes are', ({ assert }) => { const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)]) assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska', hash16k: md5(mkv).toString('hex') }) }) test('not yEnc: throws', ({ assert }) => { assert.throws(() => inspect(Buffer.from('just some text\r\n'))) }) test('at most MAX_NAMES names', ({ assert }) => { const setId = Buffer.alloc(16, 4) const par2 = Buffer.concat(Array.from({ length: MAX_NAMES + 5 }, (_, i) => fileDesc(setId, `f${i}.rar`, i))) const result = inspect(article(par2, 'p.par2')) assert.equal(result.entries, MAX_NAMES + 5) assert.lengthOf(result.names!, MAX_NAMES) }) }) test.group('probe / innerName', () => { test("the archive's largest entry, else par2's largest non-packaging file, else a lone file's yEnc name", ({ assert }) => { const archive = { yencName: 'a.rar', size: 1, type: 'rar5' as const, names: ['Show.S01E01/show.s01e01.mkv'] } const par2 = { yencName: 'a.par2', size: 1, type: 'par2' as const, names: ['abc.part01.rar', 'Show.S01E01.1080p.mkv'] } assert.equal(innerName([{ kind: 'par2', result: par2 }, { kind: 'archive', result: archive }]), 'show.s01e01.mkv') assert.equal(innerName([{ kind: 'par2', result: par2 }]), 'Show.S01E01.1080p.mkv') assert.equal(innerName([{ kind: 'par2', result: { ...par2, names: ['abc.part01.rar', 'abc.par2'] } }]), null) const lone = { kind: 'lone', result: { yencName: 'Real.Name.mkv', size: 1, type: 'matroska' as const } } assert.equal(innerName([lone], 'k3j4h5'), 'Real.Name.mkv') // A random-named file whose yEnc name is its random name says nothing new. assert.isNull(innerName([lone], 'real.name.mkv')) assert.isNull(innerName([lone], 'Real.Name')) // Nor does packaging: a lone par2 volume. assert.isNull(innerName([{ kind: 'lone', result: { yencName: 'a1b2.vol007+08.par2', size: 1, type: 'par2' } }], 'x9y8')) }) test("a 7z-tail: the listing from the last segment of the last volume; one that isn't the last segment: none", ({ assert }) => { const tail = readFileSync(new URL('../../fixtures/sevenzip/split-last-volume.tail', import.meta.url)) // The volume's last 8,000 bytes as its third part, of a volume 1,000,000 bytes long. const last = tail.subarray(tail.length - 8000) const part = (data: Buffer, begin: number) => Buffer.from( [`=ybegin part=3 total=3 line=128 size=1000000 name=x.7z.004`, `=ypart begin=${begin} end=${begin + data.length - 1}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} part=3 pcrc32=${crc32Hex(data)}`].join('\r\n'), 'latin1' ) const result = inspect(part(last, 1_000_000 - 8000 + 1), '7z-tail') assert.equal(result.entries, 4) assert.deepEqual(result.names?.[0], 'Some.Movie.2020.1080p.BluRay.x264-GRP/some.movie.2020.1080p.bluray.x264-grp.mkv') assert.isFalse(result.passworded) assert.isUndefined(inspect(part(last, 1), '7z-tail').names) const encrypted = readFileSync(new URL('../../fixtures/sevenzip/header-encrypted.tail', import.meta.url)) const hidden = inspect(part(encrypted.subarray(encrypted.length - 8000), 1_000_000 - 8000 + 1), '7z-tail') assert.isTrue(hidden.passworded) assert.equal(hidden.entries, 0) assert.isUndefined(hidden.names) // A small 7z whole in one article, probed as an archive start. assert.deepEqual(inspect(article(readFileSync(new URL('../../fixtures/sevenzip/tiny.7z', import.meta.url)), 'x.7z'), 'archive').names, ['b.nfo', 'a.txt']) }) test('a ZIP start: the names of its first entries, and whether they are encrypted', ({ assert }) => { const local = (name: string, data: Buffer, flags = 0) => { const header = Buffer.alloc(30) header.writeUInt32LE(0x04034b50, 0) header.writeUInt16LE(flags, 6) header.writeUInt32LE(data.length, 18) header.writeUInt32LE(data.length, 22) header.writeUInt16LE(Buffer.byteLength(name), 26) return Buffer.concat([header, Buffer.from(name), data]) } const zip = Buffer.concat([local('Album/', Buffer.alloc(0)), local('Album/01 - Track.flac', Buffer.alloc(3000, 1)), local('Album/cover.jpg', Buffer.alloc(500, 2))]) const plain = inspect(article(zip, 'album.zip')) assert.equal(plain.type, 'zip') assert.deepEqual(plain.names, ['Album/01 - Track.flac', 'Album/cover.jpg']) assert.isFalse(plain.passworded) assert.isTrue(inspect(article(local('secret.mkv', Buffer.alloc(100), 0x1), 'x.zip')).passworded) }) })