Daan MeijerandClaude Opus 5.5
51975fe603
One poster adds the same 22 GB hidden '.1917-….mkv' decoy to every 7z upload; the largest-entry rule named 840 releases after it. Entries with a hidden path part are skipped; release:rename --hidden renames the existing ones from their stored listing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
377 lines
17 KiB
TypeScript
377 lines
17 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { crc32 } from 'node:zlib'
|
|
import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, walkRar, type RarEntry, type VolumeReader } from '#lib/rar/index'
|
|
import { crc32Hex } from '#lib/yenc/index'
|
|
|
|
/** Entries without the extraction fields, for comparing listings. */
|
|
const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest)
|
|
|
|
// --- RAR 4 builders -----------------------------------------------------------
|
|
|
|
const RAR4 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
|
|
|
|
function rar4Block(type: number, flags: number, body: Buffer) {
|
|
const header = Buffer.alloc(7)
|
|
header.writeUInt8(type, 2)
|
|
header.writeUInt16LE(flags, 3)
|
|
header.writeUInt16LE(7 + body.length, 5)
|
|
const block = Buffer.concat([header, body])
|
|
block.writeUInt16LE(crc32(block.subarray(2)) & 0xffff, 0)
|
|
return block
|
|
}
|
|
|
|
function rar4Main(flags: number) {
|
|
return rar4Block(0x73, flags, Buffer.alloc(6))
|
|
}
|
|
|
|
function rar4File(name: Buffer, opts: { flags?: number; packed: number; unpacked: number; method?: number }) {
|
|
const body = Buffer.alloc(25)
|
|
body.writeUInt8(opts.method ?? 0x33, 18)
|
|
body.writeUInt32LE(opts.packed, 0)
|
|
body.writeUInt32LE(opts.unpacked, 4)
|
|
body.writeUInt16LE(name.length, 19)
|
|
// LONG_BLOCK (0x8000) is always set on file headers.
|
|
return Buffer.concat([rar4Block(0x74, 0x8000 | (opts.flags ?? 0), Buffer.concat([body, name])), Buffer.alloc(opts.packed)])
|
|
}
|
|
|
|
// --- RAR 5 builders -----------------------------------------------------------
|
|
|
|
const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
|
|
|
function vint(value: number) {
|
|
const bytes: number[] = []
|
|
do {
|
|
let byte = value % 128
|
|
value = Math.floor(value / 128)
|
|
if (value > 0) byte |= 0x80
|
|
bytes.push(byte)
|
|
} while (value > 0)
|
|
return Buffer.from(bytes)
|
|
}
|
|
|
|
function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
|
|
let flags = headerFlags
|
|
const parts = [vint(type)]
|
|
const sizes: Buffer[] = []
|
|
if (opts.extra) {
|
|
flags |= 0x0001
|
|
sizes.push(vint(opts.extra.length))
|
|
}
|
|
if (opts.data !== undefined) {
|
|
flags |= 0x0002
|
|
sizes.push(vint(opts.data))
|
|
}
|
|
const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
|
|
const sized = Buffer.concat([vint(header.length), header])
|
|
const crc = Buffer.alloc(4)
|
|
crc.writeUInt32LE(crc32(sized) >>> 0)
|
|
return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
|
|
}
|
|
|
|
function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
|
|
const nameBuf = Buffer.from(name, 'utf8')
|
|
const fields = Buffer.concat([
|
|
vint(opts.dir ? 0x0001 : 0), // file flags
|
|
vint(size),
|
|
vint(0x20), // attributes
|
|
vint(0), // compression info
|
|
vint(0), // host OS
|
|
vint(nameBuf.length),
|
|
nameBuf,
|
|
])
|
|
// Encryption record: size, type 1, then its data (dummy here).
|
|
const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
|
|
return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
|
|
}
|
|
|
|
test.group('rar / RAR 4', () => {
|
|
test('lists files of a first volume, including 64-bit sizes and flags', ({ assert }) => {
|
|
const data = Buffer.concat([
|
|
RAR4,
|
|
rar4Main(0x0001 | 0x0100), // volume, first volume
|
|
rar4File(Buffer.from('Some.Movie.2024/movie.mkv'), { packed: 10, unpacked: 5_000_000, flags: 0x0002 }),
|
|
])
|
|
assert.equal(rarFormat(data), 'rar4')
|
|
const info = parseRarHeaders(data)!
|
|
assert.include(info, { format: 'rar4', volume: true, firstVolume: true, headersEncrypted: false, truncated: true })
|
|
assert.deepEqual(listing(info.entries), [
|
|
{ name: 'Some.Movie.2024/movie.mkv', size: 5_000_000, packedSize: 10, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
|
|
])
|
|
})
|
|
|
|
test('detects password-protected files and encrypted headers', ({ assert }) => {
|
|
const files = parseRarHeaders(
|
|
Buffer.concat([RAR4, rar4Main(0), rar4File(Buffer.from('secret.bin'), { packed: 4, unpacked: 4, flags: 0x0004 })])
|
|
)!
|
|
assert.isTrue(files.entries[0].encrypted)
|
|
const headers = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0x0080)]))!
|
|
assert.isTrue(headers.headersEncrypted)
|
|
assert.lengthOf(headers.entries, 0)
|
|
})
|
|
|
|
test('copies out stored entries, not compressed ones', ({ assert }) => {
|
|
const data = Buffer.concat([
|
|
RAR4,
|
|
rar4Main(0),
|
|
rar4File(Buffer.from('a.nzb'), { packed: 5, unpacked: 5, method: 0x30 }),
|
|
rar4File(Buffer.from('b.nzb'), { packed: 5, unpacked: 9 }),
|
|
])
|
|
const [stored, packed] = parseRarHeaders(data)!.entries
|
|
data.write('<nzb>', stored.dataOffset)
|
|
assert.equal(storedEntryData(data, stored)?.toString(), '<nzb>')
|
|
assert.isFalse(packed.stored)
|
|
assert.isNull(storedEntryData(data, packed))
|
|
})
|
|
|
|
test('stops at a header with a bad CRC', ({ assert }) => {
|
|
const good = rar4File(Buffer.from('a.bin'), { packed: 1, unpacked: 1 })
|
|
const bad = rar4File(Buffer.from('b.bin'), { packed: 1, unpacked: 1 })
|
|
bad[10] ^= 0xff
|
|
const info = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0), good, bad]))!
|
|
assert.deepEqual(info.entries.map((e) => e.name), ['a.bin'])
|
|
})
|
|
|
|
test('decodes compressed Unicode names', ({ assert }) => {
|
|
// "é.txt": ASCII fallback "?.txt", then highByte 0, flags 0b01_00_00_00 (é via high byte,
|
|
// then 3 plain bytes), followed by one more flag byte for the last byte.
|
|
const ascii = Buffer.from('?.txt', 'latin1')
|
|
const encoded = Buffer.from([0x00, 0b01000000, 0xe9, 0x2e, 0x74, 0x78, 0b00000000, 0x74])
|
|
assert.equal(decodeRar4Name(Buffer.concat([ascii, Buffer.from([0]), encoded])), 'é.txt')
|
|
assert.equal(decodeRar4Name(Buffer.from('plain.txt')), 'plain.txt')
|
|
})
|
|
})
|
|
|
|
test.group('rar / RAR 5', () => {
|
|
test('lists files and reads volume flags', ({ assert }) => {
|
|
const data = Buffer.concat([
|
|
RAR5,
|
|
rar5Block(1, 0, vint(0x0001)), // main: volume, no volume number → first volume
|
|
rar5File('Release/track01.flac', 30_000_000, { packed: 20, headerFlags: 0x0010 }),
|
|
])
|
|
assert.equal(rarFormat(data), 'rar5')
|
|
const info = parseRarHeaders(data)!
|
|
assert.include(info, { format: 'rar5', volume: true, firstVolume: true, headersEncrypted: false })
|
|
assert.deepEqual(listing(info.entries), [
|
|
{ name: 'Release/track01.flac', size: 30_000_000, packedSize: 20, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
|
|
])
|
|
})
|
|
|
|
test('a later volume carries a volume number', ({ assert }) => {
|
|
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, Buffer.concat([vint(0x0003), vint(4)]))]))!
|
|
assert.isFalse(info.firstVolume)
|
|
})
|
|
|
|
test('detects encrypted files and encrypted headers', ({ assert }) => {
|
|
const files = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]))!
|
|
assert.isTrue(files.entries[0].encrypted)
|
|
const headers = parseRarHeaders(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]))!
|
|
assert.isTrue(headers.headersEncrypted)
|
|
})
|
|
|
|
test('marks the listing complete at the end-of-archive header', ({ assert }) => {
|
|
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a', 1, { packed: 1 }), rar5Block(5, 0, vint(0))]))!
|
|
assert.isFalse(info.truncated)
|
|
assert.lengthOf(info.entries, 1)
|
|
})
|
|
|
|
test('copies out entries stored without compression', ({ assert }) => {
|
|
const data = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a.nzb', 5, { packed: 5 }), rar5Block(5, 0, vint(0))])
|
|
const entry = parseRarHeaders(data)!.entries[0]
|
|
assert.isTrue(entry.stored)
|
|
data.write('<nzb>', entry.dataOffset)
|
|
assert.equal(storedEntryData(data, entry)?.toString(), '<nzb>')
|
|
assert.isNull(storedEntryData(data.subarray(0, entry.dataOffset + 2), entry))
|
|
assert.isNull(storedEntryData(data, { ...entry, encrypted: true }))
|
|
})
|
|
|
|
test('returns null for other data', ({ assert }) => {
|
|
assert.isNull(parseRarHeaders(Buffer.from('PAR2\0PKT', 'latin1')))
|
|
})
|
|
})
|
|
|
|
test.group('rar / release naming from contents', () => {
|
|
const info = (names: string[]) =>
|
|
({ format: 'rar5', volume: false, firstVolume: true, solid: false, headersEncrypted: false, truncated: true,
|
|
entries: names.map((name) => ({ name, size: 1, packedSize: 1, directory: false, encrypted: false, splitBefore: false, splitAfter: false })) }) as any
|
|
|
|
test('uses the shared top-level directory when it looks like a release name', async ({ assert }) => {
|
|
const { releaseNameFromContents } = await import('#services/ArchiveService')
|
|
assert.equal(
|
|
releaseNameFromContents(info(['GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/00-a.jpg', 'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/01-b.flac'])),
|
|
'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM'
|
|
)
|
|
// No directory: the largest readable entry names the release.
|
|
assert.equal(
|
|
releaseNameFromContents(info(['Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265.nzb'])),
|
|
'Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265'
|
|
)
|
|
assert.isUndefined(releaseNameFromContents(info(['movie.mkv'])))
|
|
assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91PqA7.mkv'])))
|
|
assert.isUndefined(releaseNameFromContents(info(['a/x', 'b/y'])))
|
|
assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91Pq/x.mkv'])))
|
|
})
|
|
|
|
test('hidden entries (a decoy) name nothing', async ({ assert }) => {
|
|
const { releaseNameFromContents, isHidden } = await import('#services/ArchiveService')
|
|
const decoy = '.1917-2019-2160p_MA_WEB-DL_Hybrid_H265_DV_HDR_DDP_Atmos_5.1_English-HONE-.mkv'
|
|
const listing = info([decoy, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX', `${decoy}/${decoy}.19OMnL`, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX/boston.blue.s01e02.rar'])
|
|
listing.entries[0].directory = true
|
|
listing.entries[1].directory = true
|
|
listing.entries[2].size = 22_093_429_730
|
|
assert.equal(releaseNameFromContents(listing), 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX')
|
|
assert.isTrue(isHidden(`${decoy}/x.mkv`))
|
|
assert.isTrue(isHidden('dir/.hidden.mkv'))
|
|
assert.isFalse(isHidden('Some.Release/movie.mkv'))
|
|
})
|
|
|
|
test('recognises obfuscated names', async ({ assert }) => {
|
|
const { looksRandom } = await import('#services/ArchiveService')
|
|
assert.isTrue(looksRandom('CGBWgdhlSCAOAW62gLWRRM'))
|
|
assert.isTrue(looksRandom('0220e975b1417f96b77dc64410e10ebc'))
|
|
assert.isFalse(looksRandom('Plus.and.Minus.S01E06.1080p'))
|
|
assert.isFalse(looksRandom('1790546808_nicovideo_jp_watch_sm26679715'))
|
|
})
|
|
})
|
|
|
|
test.group('rar / looksRandom capitals', () => {
|
|
test('long letter-only tokens with many capitals are random', async ({ assert }) => {
|
|
const { looksRandom } = await import('#services/ArchiveService')
|
|
assert.isTrue(looksRandom('RmISakytFwXSCEzjVEpaKk'))
|
|
assert.isTrue(looksRandom('326678d1d07ef12a.bin'))
|
|
assert.isFalse(looksRandom('Some.Show.S01E01'))
|
|
assert.isFalse(looksRandom('TheQuickBrownFoxJumps'))
|
|
})
|
|
})
|
|
|
|
// --- Archive walks (D40) --------------------------------------------------------
|
|
|
|
/** Volumes split into segments of `segmentSize` bytes; counts the segments read. */
|
|
function memoryReader(volumes: Buffer[], segmentSize: number) {
|
|
const read = new Set<string>()
|
|
const reader: VolumeReader & { segments: Set<string> } = {
|
|
segments: read,
|
|
volumes: volumes.length,
|
|
size: (v) => volumes[v].length,
|
|
async read(v, offset, more) {
|
|
const index = Math.floor(offset / segmentSize)
|
|
if (offset >= volumes[v].length) return null
|
|
read.add(`${v}:${index}`)
|
|
if (more) read.add(`${v}:${index + 1}`)
|
|
return volumes[v].subarray(offset, (index + (more ? 2 : 1)) * segmentSize)
|
|
},
|
|
}
|
|
return reader
|
|
}
|
|
|
|
function rar5Set() {
|
|
const main = (n: number) => rar5Block(1, 0, n === 0 ? vint(0x0001) : Buffer.concat([vint(0x0003), vint(n)]))
|
|
const end = rar5Block(5, 0, vint(0x0001))
|
|
const SPLIT_BEFORE = 0x0008
|
|
const SPLIT_AFTER = 0x0010
|
|
return [
|
|
Buffer.concat([RAR5, main(0), rar5File('Book/01.mp3', 3000, { packed: 3000 }), rar5File('Book/02.mp3', 7000, { packed: 5000, headerFlags: SPLIT_AFTER }), end]),
|
|
Buffer.concat([
|
|
RAR5,
|
|
main(1),
|
|
rar5File('Book/02.mp3', 7000, { packed: 2000, headerFlags: SPLIT_BEFORE }),
|
|
rar5File('Book/03.mp3', 1000, { packed: 1000 }),
|
|
rar5File('Book/04.mp3', 4500, { packed: 4000, headerFlags: SPLIT_AFTER }),
|
|
end,
|
|
]),
|
|
Buffer.concat([RAR5, main(2), rar5File('Book/04.mp3', 4500, { packed: 500, headerFlags: SPLIT_BEFORE }), rar5File('Book/05.mp3', 100, { packed: 100 }), rar5Block(5, 0, vint(0))]),
|
|
]
|
|
}
|
|
|
|
test.group('rar / archive walks', () => {
|
|
test('lists every entry of a RAR5 set, reading a segment per header', async ({ assert }) => {
|
|
// 700-byte segments: headers get cut off by segment ends.
|
|
const reader = memoryReader(rar5Set(), 700)
|
|
const walk = await walkRar(reader)
|
|
assert.isTrue(walk.complete)
|
|
assert.equal(walk.format, 'rar5')
|
|
assert.deepEqual(walk.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3', 'Book/03.mp3', 'Book/04.mp3', 'Book/05.mp3'])
|
|
assert.equal(walk.entries[1].size, 7000)
|
|
const total = rar5Set().reduce((sum, v) => sum + Math.ceil(v.length / 700), 0)
|
|
assert.isBelow(reader.segments.size, total / 2)
|
|
})
|
|
|
|
test('lists a RAR4 set across volumes', async ({ assert }) => {
|
|
const end = rar4Block(0x7b, 0, Buffer.alloc(0))
|
|
const volumes = [
|
|
Buffer.concat([RAR4, rar4Main(0x0001 | 0x0100), rar4File(Buffer.from('a.flac'), { packed: 900, unpacked: 900 }), rar4File(Buffer.from('b.flac'), { packed: 2000, unpacked: 2500, flags: 0x0002 }), end]),
|
|
Buffer.concat([RAR4, rar4Main(0x0001), rar4File(Buffer.from('b.flac'), { packed: 500, unpacked: 2500, flags: 0x0001 }), rar4File(Buffer.from('c.flac'), { packed: 10, unpacked: 10 }), end]),
|
|
]
|
|
const walk = await walkRar(memoryReader(volumes, 512))
|
|
assert.isTrue(walk.complete)
|
|
assert.deepEqual(walk.entries.map((e) => e.name), ['a.flac', 'b.flac', 'c.flac'])
|
|
})
|
|
|
|
test('stops at a missing segment or a broken header, keeping what it found', async ({ assert }) => {
|
|
const set = rar5Set()
|
|
const missing = memoryReader(set, 700)
|
|
const read = missing.read.bind(missing)
|
|
missing.read = async (v, offset, more) => (v === 1 ? null : read(v, offset, more))
|
|
const partial = await walkRar(missing)
|
|
assert.equal(partial.stopped, 'unreadable')
|
|
assert.deepEqual(partial.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3'])
|
|
|
|
const broken = rar5Set()
|
|
// The header of 03.mp3 in volume 2, after 02.mp3's continuation and its data.
|
|
const at = broken[1].indexOf(Buffer.from('Book/03.mp3')) - 4
|
|
broken[1][at] ^= 0xff
|
|
const invalid = await walkRar(memoryReader(broken, 700))
|
|
assert.equal(invalid.stopped, 'invalid')
|
|
assert.lengthOf(invalid.entries, 2)
|
|
})
|
|
|
|
test('finds segments from their yEnc offsets', async ({ assert }) => {
|
|
const { SegmentReader } = await import('#services/ArchiveWalkService')
|
|
const set = rar5Set()
|
|
const size = 700
|
|
const articles = new Map<string, Buffer>()
|
|
const files = set.map((volume, v) => {
|
|
const segments = []
|
|
for (let i = 0; i * size < volume.length; i++) {
|
|
const id = `v${v}s${i}@test`
|
|
articles.set(id, yencArticle(`x.part${v + 1}.rar`, volume, i, size))
|
|
segments.push({ number: i + 1, id, size: 0 })
|
|
}
|
|
return { id: v + 1, segments }
|
|
})
|
|
const reader = new SegmentReader(files, 100, async (id) => articles.get(id)!)
|
|
const walk = await walkRar(reader)
|
|
assert.isTrue(walk.complete)
|
|
assert.lengthOf(walk.entries, 5)
|
|
assert.isBelow(reader.fetched, articles.size / 2)
|
|
|
|
const capped = new SegmentReader(files, 3, async (id) => articles.get(id)!)
|
|
const stopped = await walkRar(capped)
|
|
assert.equal(stopped.stopped, 'unreadable')
|
|
assert.equal(capped.fetched, 3)
|
|
})
|
|
})
|
|
|
|
/** One yEnc article of `file`: part `index` of `size` bytes. */
|
|
function yencArticle(name: string, file: Buffer, index: number, size: number) {
|
|
const data = file.subarray(index * size, (index + 1) * size)
|
|
const total = Math.ceil(file.length / size)
|
|
const out: number[] = []
|
|
for (const byte of data) {
|
|
const c = (byte + 42) & 0xff
|
|
// Escape the critical characters, and '.' and whitespace so no line needs dot-stuffing.
|
|
if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
|
|
else out.push(c)
|
|
}
|
|
const lines: string[] = []
|
|
const encoded = Buffer.from(out)
|
|
for (let i = 0; i < encoded.length; i += 128) {
|
|
let end = Math.min(encoded.length, i + 128)
|
|
if (encoded[end - 1] === 0x3d && end < encoded.length) end++
|
|
lines.push(encoded.subarray(i, end).toString('latin1'))
|
|
i = end - 128
|
|
}
|
|
const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
|
|
const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
|
|
return Buffer.from(head + lines.join('\r\n') + tail, 'latin1')
|
|
}
|