Rebuild: phase 2's prober reads the probe queue
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
d72fe2f13f
commit
8bd6d6aa5f
30 files changed
+1666
-7
No files matched your search
@@ -0,0 +1,124 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { crc32 } from 'node:zlib'
|
||||
import yencode from 'yencode'
|
||||
import { crc32Hex } from '#lib/yenc/index'
|
||||
import { inspect, innerName, MAX_NAMES } from '#lib/probe/index'
|
||||
|
||||
/** A single-part yEnc article of `data`. */
|
||||
function article(data: Buffer, name: string) {
|
||||
const lines = [`=ybegin line=128 size=${data.length} name=${name}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} crc32=${crc32Hex(data)}`]
|
||||
return Buffer.from(lines.join('\r\n'), 'latin1')
|
||||
}
|
||||
|
||||
const md5 = (data: Buffer) => createHash('md5').update(data).digest()
|
||||
/** Builds one PAR2 packet with a valid checksum. */
|
||||
function packet(setId: Buffer, type: string, body: Buffer) {
|
||||
const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
|
||||
const length = Buffer.alloc(8)
|
||||
length.writeBigUInt64LE(BigInt(64 + padded.length))
|
||||
const typeBuf = Buffer.alloc(16)
|
||||
typeBuf.write(type, 'latin1')
|
||||
const hashed = Buffer.concat([setId, typeBuf, padded])
|
||||
return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
|
||||
}
|
||||
function fileDesc(setId: Buffer, name: string, size: number) {
|
||||
const len = Buffer.alloc(8)
|
||||
len.writeBigUInt64LE(BigInt(size))
|
||||
const md5_16k = md5(Buffer.from(`${name}-16k`))
|
||||
const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
|
||||
const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
|
||||
return packet(setId, 'PAR 2.0\0FileDesc', body)
|
||||
}
|
||||
const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
||||
function vint(value: number) {
|
||||
const bytes: number[] = []
|
||||
do {
|
||||
let byte = value % 128
|
||||
value = Math.floor(value / 128)
|
||||
if (value > 0) byte |= 0x80
|
||||
bytes.push(byte)
|
||||
} while (value > 0)
|
||||
return Buffer.from(bytes)
|
||||
}
|
||||
function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
|
||||
let flags = headerFlags
|
||||
const parts = [vint(type)]
|
||||
const sizes: Buffer[] = []
|
||||
if (opts.extra) {
|
||||
flags |= 0x0001
|
||||
sizes.push(vint(opts.extra.length))
|
||||
}
|
||||
if (opts.data !== undefined) {
|
||||
flags |= 0x0002
|
||||
sizes.push(vint(opts.data))
|
||||
}
|
||||
const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
|
||||
const sized = Buffer.concat([vint(header.length), header])
|
||||
const crc = Buffer.alloc(4)
|
||||
crc.writeUInt32LE(crc32(sized) >>> 0)
|
||||
return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
|
||||
}
|
||||
function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
|
||||
const nameBuf = Buffer.from(name, 'utf8')
|
||||
const fields = Buffer.concat([
|
||||
vint(opts.dir ? 0x0001 : 0), // file flags
|
||||
vint(size),
|
||||
vint(0x20), // attributes
|
||||
vint(0), // compression info
|
||||
vint(0), // host OS
|
||||
vint(nameBuf.length),
|
||||
nameBuf,
|
||||
])
|
||||
// Encryption record: size, type 1, then its data (dummy here).
|
||||
const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
|
||||
return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
|
||||
}
|
||||
|
||||
test.group('probe / inspect', () => {
|
||||
test('a par2 index: its files, largest first', ({ assert }) => {
|
||||
const setId = Buffer.alloc(16, 3)
|
||||
const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)])
|
||||
const result = inspect(article(par2, 'a8f3k2.par2'))
|
||||
assert.deepEqual(result, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
|
||||
})
|
||||
|
||||
test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => {
|
||||
const rar = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('Movie.2020.1080p/movie.mkv', 5_000_000, { packed: 10 }), rar5File('Movie.2020.1080p/sample.mkv', 1000, { packed: 10 })])
|
||||
const open = inspect(article(rar, 'x7q.part01.rar'))
|
||||
assert.include(open, { type: 'rar5', entries: 2, passworded: false })
|
||||
assert.deepEqual(open.names, ['Movie.2020.1080p/movie.mkv', 'Movie.2020.1080p/sample.mkv'])
|
||||
const locked = inspect(article(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]), 'y.rar'))
|
||||
assert.isTrue(locked.passworded)
|
||||
const hidden = inspect(article(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]), 'z.rar'))
|
||||
assert.include(hidden, { passworded: true, entries: 0 })
|
||||
})
|
||||
|
||||
test('other data: the yEnc name and what the bytes are', ({ assert }) => {
|
||||
const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)])
|
||||
assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska' })
|
||||
})
|
||||
|
||||
test('not yEnc: throws', ({ assert }) => {
|
||||
assert.throws(() => inspect(Buffer.from('just some text\r\n')))
|
||||
})
|
||||
|
||||
test('at most MAX_NAMES names', ({ assert }) => {
|
||||
const setId = Buffer.alloc(16, 4)
|
||||
const par2 = Buffer.concat(Array.from({ length: MAX_NAMES + 5 }, (_, i) => fileDesc(setId, `f${i}.rar`, i)))
|
||||
const result = inspect(article(par2, 'p.par2'))
|
||||
assert.equal(result.entries, MAX_NAMES + 5)
|
||||
assert.lengthOf(result.names!, MAX_NAMES)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('probe / innerName', () => {
|
||||
test("the archive's largest entry, else par2's largest non-packaging file, else a lone file's yEnc name", ({ assert }) => {
|
||||
const archive = { yencName: 'a.rar', size: 1, type: 'rar5' as const, names: ['Show.S01E01/show.s01e01.mkv'] }
|
||||
const par2 = { yencName: 'a.par2', size: 1, type: 'par2' as const, names: ['abc.part01.rar', 'Show.S01E01.1080p.mkv'] }
|
||||
assert.equal(innerName({ archive, par2 }), 'show.s01e01.mkv')
|
||||
assert.equal(innerName({ par2 }), 'Show.S01E01.1080p.mkv')
|
||||
assert.equal(innerName({ par2: { ...par2, names: ['abc.part01.rar', 'abc.par2'] } }), null)
|
||||
assert.equal(innerName({ lone: { yencName: 'Real.Name.mkv', size: 1, type: 'matroska' } }), 'Real.Name.mkv')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,60 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { isPar2, parsePar2 } from '#lib/par2/index'
|
||||
|
||||
const md5 = (data: Buffer) => createHash('md5').update(data).digest()
|
||||
|
||||
/** Builds one PAR2 packet with a valid checksum. */
|
||||
function packet(setId: Buffer, type: string, body: Buffer) {
|
||||
const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
|
||||
const length = Buffer.alloc(8)
|
||||
length.writeBigUInt64LE(BigInt(64 + padded.length))
|
||||
const typeBuf = Buffer.alloc(16)
|
||||
typeBuf.write(type, 'latin1')
|
||||
const hashed = Buffer.concat([setId, typeBuf, padded])
|
||||
return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
|
||||
}
|
||||
|
||||
function fileDesc(setId: Buffer, name: string, size: number) {
|
||||
const len = Buffer.alloc(8)
|
||||
len.writeBigUInt64LE(BigInt(size))
|
||||
const md5_16k = md5(Buffer.from(`${name}-16k`))
|
||||
const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
|
||||
const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
|
||||
return packet(setId, 'PAR 2.0\0FileDesc', body)
|
||||
}
|
||||
|
||||
test.group('par2 / parsePar2', () => {
|
||||
const setId = Buffer.alloc(16, 7)
|
||||
|
||||
test('reads file names and sizes from FileDesc packets', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
packet(setId, 'PAR 2.0\0Main', Buffer.alloc(12)),
|
||||
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600),
|
||||
fileDesc(setId, 'Some.Movie.2024.part02.rar', 52428800),
|
||||
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), // duplicates are common
|
||||
])
|
||||
assert.isTrue(isPar2(data))
|
||||
const set = parsePar2(data)!
|
||||
assert.equal(set.recoverySetId, setId.toString('hex'))
|
||||
assert.deepEqual(
|
||||
set.files.map((f) => [f.name, f.length]),
|
||||
[
|
||||
['Some.Movie.2024.part01.rar', 104857600],
|
||||
['Some.Movie.2024.part02.rar', 52428800],
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
test('skips corrupted packets and resumes at the next one', ({ assert }) => {
|
||||
const bad = fileDesc(setId, 'bad.bin', 1)
|
||||
bad[100] ^= 0xff
|
||||
const data = Buffer.concat([bad, fileDesc(setId, 'good.bin', 12345)])
|
||||
assert.deepEqual(parsePar2(data)!.files.map((f) => f.name), ['good.bin'])
|
||||
})
|
||||
|
||||
test('returns null for non-par2 data', ({ assert }) => {
|
||||
assert.isFalse(isPar2(Buffer.from('Rar!\x1a\x07\x00', 'latin1')))
|
||||
assert.isNull(parsePar2(Buffer.from('hello world')))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,189 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { crc32 } from 'node:zlib'
|
||||
import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, type RarEntry } from '#lib/rar/index'
|
||||
|
||||
/** Entries without the extraction fields, for comparing listings. */
|
||||
const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest)
|
||||
|
||||
// --- RAR 4 builders -----------------------------------------------------------
|
||||
|
||||
const RAR4 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
|
||||
|
||||
function rar4Block(type: number, flags: number, body: Buffer) {
|
||||
const header = Buffer.alloc(7)
|
||||
header.writeUInt8(type, 2)
|
||||
header.writeUInt16LE(flags, 3)
|
||||
header.writeUInt16LE(7 + body.length, 5)
|
||||
const block = Buffer.concat([header, body])
|
||||
block.writeUInt16LE(crc32(block.subarray(2)) & 0xffff, 0)
|
||||
return block
|
||||
}
|
||||
|
||||
function rar4Main(flags: number) {
|
||||
return rar4Block(0x73, flags, Buffer.alloc(6))
|
||||
}
|
||||
|
||||
function rar4File(name: Buffer, opts: { flags?: number; packed: number; unpacked: number; method?: number }) {
|
||||
const body = Buffer.alloc(25)
|
||||
body.writeUInt8(opts.method ?? 0x33, 18)
|
||||
body.writeUInt32LE(opts.packed, 0)
|
||||
body.writeUInt32LE(opts.unpacked, 4)
|
||||
body.writeUInt16LE(name.length, 19)
|
||||
// LONG_BLOCK (0x8000) is always set on file headers.
|
||||
return Buffer.concat([rar4Block(0x74, 0x8000 | (opts.flags ?? 0), Buffer.concat([body, name])), Buffer.alloc(opts.packed)])
|
||||
}
|
||||
|
||||
// --- RAR 5 builders -----------------------------------------------------------
|
||||
|
||||
const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
||||
|
||||
function vint(value: number) {
|
||||
const bytes: number[] = []
|
||||
do {
|
||||
let byte = value % 128
|
||||
value = Math.floor(value / 128)
|
||||
if (value > 0) byte |= 0x80
|
||||
bytes.push(byte)
|
||||
} while (value > 0)
|
||||
return Buffer.from(bytes)
|
||||
}
|
||||
|
||||
function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
|
||||
let flags = headerFlags
|
||||
const parts = [vint(type)]
|
||||
const sizes: Buffer[] = []
|
||||
if (opts.extra) {
|
||||
flags |= 0x0001
|
||||
sizes.push(vint(opts.extra.length))
|
||||
}
|
||||
if (opts.data !== undefined) {
|
||||
flags |= 0x0002
|
||||
sizes.push(vint(opts.data))
|
||||
}
|
||||
const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
|
||||
const sized = Buffer.concat([vint(header.length), header])
|
||||
const crc = Buffer.alloc(4)
|
||||
crc.writeUInt32LE(crc32(sized) >>> 0)
|
||||
return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
|
||||
}
|
||||
|
||||
function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
|
||||
const nameBuf = Buffer.from(name, 'utf8')
|
||||
const fields = Buffer.concat([
|
||||
vint(opts.dir ? 0x0001 : 0), // file flags
|
||||
vint(size),
|
||||
vint(0x20), // attributes
|
||||
vint(0), // compression info
|
||||
vint(0), // host OS
|
||||
vint(nameBuf.length),
|
||||
nameBuf,
|
||||
])
|
||||
// Encryption record: size, type 1, then its data (dummy here).
|
||||
const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
|
||||
return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
|
||||
}
|
||||
|
||||
test.group('rar / RAR 4', () => {
|
||||
test('lists files of a first volume, including 64-bit sizes and flags', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
RAR4,
|
||||
rar4Main(0x0001 | 0x0100), // volume, first volume
|
||||
rar4File(Buffer.from('Some.Movie.2024/movie.mkv'), { packed: 10, unpacked: 5_000_000, flags: 0x0002 }),
|
||||
])
|
||||
assert.equal(rarFormat(data), 'rar4')
|
||||
const info = parseRarHeaders(data)!
|
||||
assert.include(info, { format: 'rar4', volume: true, firstVolume: true, headersEncrypted: false, truncated: true })
|
||||
assert.deepEqual(listing(info.entries), [
|
||||
{ name: 'Some.Movie.2024/movie.mkv', size: 5_000_000, packedSize: 10, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
|
||||
])
|
||||
})
|
||||
|
||||
test('detects password-protected files and encrypted headers', ({ assert }) => {
|
||||
const files = parseRarHeaders(
|
||||
Buffer.concat([RAR4, rar4Main(0), rar4File(Buffer.from('secret.bin'), { packed: 4, unpacked: 4, flags: 0x0004 })])
|
||||
)!
|
||||
assert.isTrue(files.entries[0].encrypted)
|
||||
const headers = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0x0080)]))!
|
||||
assert.isTrue(headers.headersEncrypted)
|
||||
assert.lengthOf(headers.entries, 0)
|
||||
})
|
||||
|
||||
test('copies out stored entries, not compressed ones', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
RAR4,
|
||||
rar4Main(0),
|
||||
rar4File(Buffer.from('a.nzb'), { packed: 5, unpacked: 5, method: 0x30 }),
|
||||
rar4File(Buffer.from('b.nzb'), { packed: 5, unpacked: 9 }),
|
||||
])
|
||||
const [stored, packed] = parseRarHeaders(data)!.entries
|
||||
data.write('<nzb>', stored.dataOffset)
|
||||
assert.equal(storedEntryData(data, stored)?.toString(), '<nzb>')
|
||||
assert.isFalse(packed.stored)
|
||||
assert.isNull(storedEntryData(data, packed))
|
||||
})
|
||||
|
||||
test('stops at a header with a bad CRC', ({ assert }) => {
|
||||
const good = rar4File(Buffer.from('a.bin'), { packed: 1, unpacked: 1 })
|
||||
const bad = rar4File(Buffer.from('b.bin'), { packed: 1, unpacked: 1 })
|
||||
bad[10] ^= 0xff
|
||||
const info = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0), good, bad]))!
|
||||
assert.deepEqual(info.entries.map((e) => e.name), ['a.bin'])
|
||||
})
|
||||
|
||||
test('decodes compressed Unicode names', ({ assert }) => {
|
||||
// "é.txt": ASCII fallback "?.txt", then highByte 0, flags 0b01_00_00_00 (é via high byte,
|
||||
// then 3 plain bytes), followed by one more flag byte for the last byte.
|
||||
const ascii = Buffer.from('?.txt', 'latin1')
|
||||
const encoded = Buffer.from([0x00, 0b01000000, 0xe9, 0x2e, 0x74, 0x78, 0b00000000, 0x74])
|
||||
assert.equal(decodeRar4Name(Buffer.concat([ascii, Buffer.from([0]), encoded])), 'é.txt')
|
||||
assert.equal(decodeRar4Name(Buffer.from('plain.txt')), 'plain.txt')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('rar / RAR 5', () => {
|
||||
test('lists files and reads volume flags', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
RAR5,
|
||||
rar5Block(1, 0, vint(0x0001)), // main: volume, no volume number → first volume
|
||||
rar5File('Release/track01.flac', 30_000_000, { packed: 20, headerFlags: 0x0010 }),
|
||||
])
|
||||
assert.equal(rarFormat(data), 'rar5')
|
||||
const info = parseRarHeaders(data)!
|
||||
assert.include(info, { format: 'rar5', volume: true, firstVolume: true, headersEncrypted: false })
|
||||
assert.deepEqual(listing(info.entries), [
|
||||
{ name: 'Release/track01.flac', size: 30_000_000, packedSize: 20, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
|
||||
])
|
||||
})
|
||||
|
||||
test('a later volume carries a volume number', ({ assert }) => {
|
||||
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, Buffer.concat([vint(0x0003), vint(4)]))]))!
|
||||
assert.isFalse(info.firstVolume)
|
||||
})
|
||||
|
||||
test('detects encrypted files and encrypted headers', ({ assert }) => {
|
||||
const files = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]))!
|
||||
assert.isTrue(files.entries[0].encrypted)
|
||||
const headers = parseRarHeaders(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]))!
|
||||
assert.isTrue(headers.headersEncrypted)
|
||||
})
|
||||
|
||||
test('marks the listing complete at the end-of-archive header', ({ assert }) => {
|
||||
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a', 1, { packed: 1 }), rar5Block(5, 0, vint(0))]))!
|
||||
assert.isFalse(info.truncated)
|
||||
assert.lengthOf(info.entries, 1)
|
||||
})
|
||||
|
||||
test('copies out entries stored without compression', ({ assert }) => {
|
||||
const data = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a.nzb', 5, { packed: 5 }), rar5Block(5, 0, vint(0))])
|
||||
const entry = parseRarHeaders(data)!.entries[0]
|
||||
assert.isTrue(entry.stored)
|
||||
data.write('<nzb>', entry.dataOffset)
|
||||
assert.equal(storedEntryData(data, entry)?.toString(), '<nzb>')
|
||||
assert.isNull(storedEntryData(data.subarray(0, entry.dataOffset + 2), entry))
|
||||
assert.isNull(storedEntryData(data, { ...entry, encrypted: true }))
|
||||
})
|
||||
|
||||
test('returns null for other data', ({ assert }) => {
|
||||
assert.isNull(parseRarHeaders(Buffer.from('PAR2\0PKT', 'latin1')))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
import { test } from '@japa/runner'
|
||||
import yencode from 'yencode'
|
||||
import {
|
||||
YencCrcError,
|
||||
YencError,
|
||||
crc32Hex,
|
||||
decodeSegment,
|
||||
parseYencMeta,
|
||||
} from '#lib/yenc/index'
|
||||
|
||||
/** Builds an article body as a yEnc poster would send it. */
|
||||
function article(
|
||||
data: Buffer,
|
||||
opts: { name: string; fileSize?: number; part?: number; total?: number; begin?: number; fileCrc?: string }
|
||||
) {
|
||||
const fileSize = opts.fileSize ?? data.length
|
||||
const encoded = yencode.encode(data).toString('latin1')
|
||||
const lines: string[] = []
|
||||
if (opts.part) {
|
||||
lines.push(`=ybegin part=${opts.part} total=${opts.total} line=128 size=${fileSize} name=${opts.name}`)
|
||||
const begin = opts.begin ?? 1
|
||||
lines.push(`=ypart begin=${begin} end=${begin + data.length - 1}`)
|
||||
lines.push(encoded)
|
||||
const fileCrc = opts.fileCrc ? ` crc32=${opts.fileCrc}` : ''
|
||||
lines.push(`=yend size=${data.length} part=${opts.part} pcrc32=${crc32Hex(data)}${fileCrc}`)
|
||||
} else {
|
||||
lines.push(`=ybegin line=128 size=${fileSize} name=${opts.name}`)
|
||||
lines.push(encoded)
|
||||
lines.push(`=yend size=${data.length} crc32=${crc32Hex(data)}`)
|
||||
}
|
||||
return Buffer.from(lines.join('\r\n'), 'latin1')
|
||||
}
|
||||
|
||||
function randomBytes(length: number, seed = 1) {
|
||||
const buf = Buffer.alloc(length)
|
||||
let x = seed
|
||||
for (let i = 0; i < length; i++) {
|
||||
x = (x * 1103515245 + 12345) & 0x7fffffff
|
||||
buf[i] = x & 0xff
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
test.group('yenc / parseYencMeta', () => {
|
||||
test('parses a multi-part header as seen on alt.binaries.test', ({ assert }) => {
|
||||
const body = Buffer.from(
|
||||
'=ybegin part=56 line=128 size=104857600 name=obrMkrDeLmbhaouwYO34.part08.rar\r\n' +
|
||||
'=ypart begin=21120001 end=21504000\r\n' +
|
||||
'xyz\r\n' +
|
||||
'=yend size=384000 part=56 pcrc32=B6339AFC',
|
||||
'latin1'
|
||||
)
|
||||
const meta = parseYencMeta(body)
|
||||
assert.deepEqual(meta.begin, {
|
||||
part: 56,
|
||||
total: undefined,
|
||||
line: 128,
|
||||
size: 104857600,
|
||||
name: 'obrMkrDeLmbhaouwYO34.part08.rar',
|
||||
})
|
||||
assert.deepEqual(meta.part, { begin: 21120001, end: 21504000 })
|
||||
assert.deepEqual(meta.end, { size: 384000, part: 56, pcrc32: 'b6339afc', crc32: undefined })
|
||||
assert.equal(body.subarray(meta.dataStart, meta.dataEnd).toString(), 'xyz')
|
||||
})
|
||||
|
||||
test('keeps spaces and "=" in the filename', ({ assert }) => {
|
||||
const body = article(Buffer.from('hi'), { name: 'My File = v2 (final).nfo' })
|
||||
assert.equal(parseYencMeta(body).begin.name, 'My File = v2 (final).nfo')
|
||||
})
|
||||
|
||||
test('ignores leading blank lines before =ybegin', ({ assert }) => {
|
||||
const body = Buffer.concat([Buffer.from('\r\n\r\n'), article(Buffer.from('hi'), { name: 'a.txt' })])
|
||||
assert.equal(parseYencMeta(body).begin.name, 'a.txt')
|
||||
})
|
||||
|
||||
test('rejects bodies without yEnc markers', ({ assert }) => {
|
||||
assert.throws(() => parseYencMeta(Buffer.from('plain text\r\n')), /=ybegin/)
|
||||
assert.throws(() => parseYencMeta(Buffer.from('=ybegin line=128 size=2 name=a\r\nxx')), /=yend/)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('yenc / decodeSegment', () => {
|
||||
test('round-trips a single-part post and checks crc32', ({ assert }) => {
|
||||
const data = randomBytes(5000)
|
||||
const segment = decodeSegment(article(data, { name: 'single.bin' }))
|
||||
assert.isTrue(segment.data.equals(data))
|
||||
assert.equal(segment.offset, 0)
|
||||
})
|
||||
|
||||
test('throws YencCrcError on a corrupted part', ({ assert }) => {
|
||||
const body = article(randomBytes(1000), { name: 'x.bin', part: 1, total: 1 })
|
||||
const index = body.indexOf('\r\n=ypart') + 60
|
||||
body[index] = body[index] === 0x41 ? 0x42 : 0x41
|
||||
assert.throws(() => decodeSegment(body), YencCrcError as any)
|
||||
})
|
||||
|
||||
test('undoes NNTP dot-stuffing', ({ assert }) => {
|
||||
// Byte 4 encodes to "." (4 + 42 = 46). An unescaped leading "." is dot-stuffed to "..".
|
||||
const data = Buffer.from([4, 5, 6])
|
||||
const body = Buffer.from(
|
||||
`=ybegin line=128 size=3 name=dot.bin\r\n../0\r\n=yend size=3 crc32=${crc32Hex(data)}`,
|
||||
'latin1'
|
||||
)
|
||||
assert.isTrue(decodeSegment(body).data.equals(data))
|
||||
assert.throws(() => decodeSegment(body, { stripDots: false }), YencError as any)
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user