Daan MeijerandClaude Opus 5.5
8bd6d6aa5f
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
2.4 KiB
TypeScript
75 lines
2.4 KiB
TypeScript
import { createHash } from 'node:crypto'
|
|
|
|
export interface Par2File {
|
|
/** File ID (hex): MD5 of md5_16k, length and name, as defined by PAR2. */
|
|
fileId: string
|
|
/** MD5 of the whole file (hex). */
|
|
md5: string
|
|
/** MD5 of the first 16 KiB (hex). */
|
|
md5_16k: string
|
|
length: number
|
|
name: string
|
|
}
|
|
|
|
export interface Par2Set {
|
|
/** Recovery set ID (hex), shared by every packet of one par2 set. */
|
|
recoverySetId: string
|
|
files: Par2File[]
|
|
}
|
|
|
|
const MAGIC = Buffer.from('PAR2\0PKT', 'latin1')
|
|
const HEADER_SIZE = 64
|
|
const FILE_DESC = Buffer.from('PAR 2.0\0FileDesc', 'latin1')
|
|
|
|
export function isPar2(data: Buffer): boolean {
|
|
return data.subarray(0, MAGIC.length).equals(MAGIC)
|
|
}
|
|
|
|
/**
|
|
* Reads the FileDesc packets of a par2 file (index or recovery volume).
|
|
* Packets with a bad length or checksum are skipped; parsing resumes at the
|
|
* next magic sequence. Returns null when no valid FileDesc packet is found.
|
|
*/
|
|
export function parsePar2(data: Buffer): Par2Set | null {
|
|
const files = new Map<string, Par2File>()
|
|
let recoverySetId: string | undefined
|
|
let offset = data.indexOf(MAGIC)
|
|
|
|
while (offset !== -1 && offset + HEADER_SIZE <= data.length) {
|
|
const length = Number(data.readBigUInt64LE(offset + 8))
|
|
const valid =
|
|
length >= HEADER_SIZE &&
|
|
length % 4 === 0 &&
|
|
offset + length <= data.length &&
|
|
createHash('md5')
|
|
.update(data.subarray(offset + 32, offset + length))
|
|
.digest()
|
|
.equals(data.subarray(offset + 16, offset + 32))
|
|
|
|
if (!valid) {
|
|
offset = data.indexOf(MAGIC, offset + 1)
|
|
continue
|
|
}
|
|
|
|
const setId = data.subarray(offset + 32, offset + 48).toString('hex')
|
|
const type = data.subarray(offset + 48, offset + 64)
|
|
if (type.equals(FILE_DESC) && (!recoverySetId || recoverySetId === setId)) {
|
|
recoverySetId = setId
|
|
const body = data.subarray(offset + HEADER_SIZE, offset + length)
|
|
const fileId = body.subarray(0, 16).toString('hex')
|
|
files.set(fileId, {
|
|
fileId,
|
|
md5: body.subarray(16, 32).toString('hex'),
|
|
md5_16k: body.subarray(32, 48).toString('hex'),
|
|
length: Number(body.readBigUInt64LE(48)),
|
|
// The name is padded with NULs to a multiple of 4 bytes.
|
|
name: body.subarray(56).toString('utf8').replace(/\0+$/, ''),
|
|
})
|
|
}
|
|
offset += length
|
|
}
|
|
|
|
if (!recoverySetId || files.size === 0) return null
|
|
return { recoverySetId, files: [...files.values()] }
|
|
}
|