import { createHash } from 'node:crypto' import { decodeSegment, YencCrcError, YencError, YencFile, type DecodedSegment } from '../yenc/index.js' import { isPar2, parsePar2, type Par2Set } from '../par2/index.js' import { parseRarHeaders, type RarEntry, type RarInfo } from '../rar/parse.js' import { extractEntry } from '../rar/extract.js' import { walkRar } from '../rar/walk.js' import { SegmentReader, type BodyFetcher } from '../rar/segment_reader.js' import { listSevenZip } from '../sevenzip/list.js' import { sniffContentType, type ContentType } from './sniff.js' import { NICOVIDEO_FILE, nicovideoInfo, zstdPrefix, type NicovideoInfo } from './nicovideo.js' /** * Content inspection's download-and-parse half (CONCEPT.md D21, D41), * without a database: what the indexer needs to know about a file, from its * segments. The indexer prepares a task from the database, the probe runs * here or on a node (articles through the seed API's relay), and the * indexer applies the result. Tasks and results are plain JSON. */ export interface TaskSegment { number: number id: string /** Article size in bytes. */ size?: number } /** Inspect one file: its first segment, and more where its kind needs it. */ export interface InspectTask { kind: 'inspect' fileId: number /** real_name, else filename. */ name: string segments: TaskSegment[] /** For a 7z listing: the archive's volume count and its last volume's segments; null when volumes are missing. */ sevenZip: { volumes: number; last: TaskSegment[] } | null } export interface InspectProbe { /** The whole file's size, from the yEnc header. */ size: number contentType: ContentType | null md5_16k: string | null /** The file name in the yEnc header. */ yencName: string nicovideo?: NicovideoInfo | null /** A whole small SFV (latin1) or NZB (utf8). */ text?: string | null par2?: Par2Set | null archive?: RarInfo | null /** NZBs read from inside an unsplit archive, and how many NZB entries it lists. */ archiveNzbs?: { entryName: string; xml: string }[] archiveNzbEntries?: number } /** Walk a RAR set's headers for its complete listing (D40). */ export interface WalkTask { kind: 'walk' releaseId: number volumes: TaskSegment[][] maxSegments: number } export interface WalkProbe { entries: Omit[] complete: boolean stopped?: string headersEncrypted: boolean segments: number } export type ContentTask = InspectTask | WalkTask /** Small posted files (NZBs, SFVs) that are read completely. */ const WHOLE_MAX_PARTS = 40 const WHOLE_MAX_BYTES = 25 * 1024 * 1024 /** Par2 index files are small; recovery volumes can be large and are not needed. */ const PAR2_MAX_PARTS = 20 const FIRST_CHUNK = 16 * 1024 /** MD5 of the first 16 KiB of a file, as par2 records it; undefined when the data is shorter than that but the file isn't. */ export function chunkMd5(data: Buffer, fileSize: number): string | undefined { if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex') } /** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */ export function decodeBody(body: Buffer): DecodedSegment { try { return decodeSegment(body) } catch (error) { if (!(error instanceof YencError) || error instanceof YencCrcError) throw error return decodeSegment(body, { lenientSize: true }) } } /** All of a (small) file whose first segment is already decoded; null when it is too big. */ export async function readWhole(first: Buffer, segments: TaskSegment[], fetch: BodyFetcher): Promise { const bytes = segments.reduce((sum, p) => sum + (p.size ?? 0), 0) if (segments.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null if (segments.length === 1) return first let assembled: YencFile | undefined for (const part of segments) { const segment = decodeSegment(await fetch(part.id), { lenientSize: true }) assembled ??= YencFile.fromSegment(segment) assembled.add(segment) } return assembled!.getBuffer() } /** A par2 file whose first segment is already decoded, with its other parts if any; null when it is not par2. */ export async function readPar2(first: Buffer, segments: TaskSegment[], fetch: BodyFetcher): Promise { if (!isPar2(first)) return null if (segments.length > PAR2_MAX_PARTS) return null if (segments.length === 1) return parsePar2(first) let assembled: YencFile | undefined for (const part of segments) { const segment = decodeSegment(await fetch(part.id)) assembled ??= YencFile.fromSegment(segment) assembled.add(segment) } return parsePar2(assembled!.getBuffer()) } export function isNzbEntry(entry: Pick): boolean { return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory } /** NZBs that can be read from a single, unencrypted archive. */ function hasNzbEntries(info: RarInfo): boolean { return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry) } export async function probeInspection( task: InspectTask, fetch: BodyFetcher, warn: (message: string) => void = () => {} ): Promise { const segment = decodeBody(await fetch(task.segments[0].id)) const size = segment.meta.begin.size const contentType = sniffContentType(segment.data) const probe: InspectProbe = { size, contentType, md5_16k: chunkMd5(segment.data, size) ?? null, yencName: segment.meta.begin.name, } // An encrypted NZB, rclone crypt or yEncBin's container: nothing to read. if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') return probe if (NICOVIDEO_FILE.test(task.name)) { probe.nicovideo = nicovideoInfo(await zstdPrefix(segment.data)) return probe } if (/\.sfv$/i.test(task.name)) { probe.text = (await readWhole(segment.data, task.segments, fetch))?.toString('latin1') ?? null return probe } if (contentType === 'nzb') { probe.text = (await readWhole(segment.data, task.segments, fetch))?.toString('utf8') ?? null return probe } probe.par2 = await readPar2(segment.data, task.segments, fetch) if (probe.par2) return probe if (contentType === '7z') { probe.archive = task.sevenZip ? await listSevenZip(segment.data, task.sevenZip.volumes, task.sevenZip.last, fetch, warn) : null return probe } const info = parseRarHeaders(segment.data) probe.archive = info if (info && hasNzbEntries(info)) { probe.archiveNzbs = [] const archive = await readWhole(segment.data, task.segments, fetch) const archived = archive && parseRarHeaders(archive) const entries = archived ? archived.entries.filter(isNzbEntry) : [] probe.archiveNzbEntries = entries.length for (const entry of entries) { const xml = await extractEntry(archive!, entry, undefined, warn) if (xml) probe.archiveNzbs.push({ entryName: entry.name, xml: xml.toString('utf8') }) } } return probe } export async function probeWalk(task: WalkTask, fetch: BodyFetcher): Promise { const reader = new SegmentReader(task.volumes, task.maxSegments, fetch) const walked = await walkRar(reader) return { entries: walked.entries.map(({ stored: _s, dataOffset: _d, ...entry }) => entry), complete: walked.complete, stopped: walked.stopped, headersEncrypted: walked.headersEncrypted, segments: reader.fetched, } }