Daan MeijerandClaude Opus 5.5
8bd6d6aa5f
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
77 lines
2.6 KiB
TypeScript
77 lines
2.6 KiB
TypeScript
import yencode from 'yencode'
|
|
import { YencCrcError, YencError } from './errors.js'
|
|
import { parseYencMeta, type YencMeta } from './meta.js'
|
|
|
|
export interface DecodedSegment {
|
|
meta: YencMeta
|
|
/** Decoded bytes of this segment. */
|
|
data: Buffer
|
|
/** 0-based offset of `data` within the complete file. */
|
|
offset: number
|
|
}
|
|
|
|
export interface DecodeOptions {
|
|
/**
|
|
* Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw
|
|
* bodies, so this defaults to true.
|
|
*/
|
|
stripDots?: boolean
|
|
/** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */
|
|
verifyCrc?: boolean
|
|
/**
|
|
* Accept a decoded length that differs from the declared size (the CRC is
|
|
* still checked). Some posting tools declare a size that leaves out a
|
|
* header, e.g. the 20-byte header of `nzbe` containers.
|
|
*/
|
|
lenientSize?: boolean
|
|
}
|
|
|
|
export function crc32Hex(data: Buffer): string {
|
|
return yencode.crc32(data).toString('hex')
|
|
}
|
|
|
|
/**
|
|
* Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order,
|
|
* without the data itself.
|
|
*/
|
|
export function combineCrc32(parts: { crc: string; length: number }[]): string {
|
|
let combined: Buffer | undefined
|
|
for (const part of parts) {
|
|
const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex')
|
|
combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc
|
|
}
|
|
return (combined ?? Buffer.alloc(4)).toString('hex')
|
|
}
|
|
|
|
/**
|
|
* Decodes one yEnc article body and checks its size and CRC32 against the
|
|
* `=yend` trailer.
|
|
*/
|
|
export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment {
|
|
const { stripDots = true, verifyCrc = true, lenientSize = false } = options
|
|
const meta = parseYencMeta(body)
|
|
const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots)
|
|
|
|
if (lenientSize) {
|
|
// Only the CRC below decides.
|
|
} else if (data.length !== meta.end.size) {
|
|
throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`)
|
|
}
|
|
if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) {
|
|
throw new YencError(
|
|
`Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}`
|
|
)
|
|
}
|
|
|
|
if (verifyCrc) {
|
|
// Multi-part posts carry pcrc32 per part; single-part posts only crc32.
|
|
const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32)
|
|
if (expected) {
|
|
const actual = crc32Hex(data)
|
|
if (actual !== expected) throw new YencCrcError(expected, actual)
|
|
}
|
|
}
|
|
|
|
return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 }
|
|
}
|