import { parseXrefGroups } from '../nntp/xref.js' import { fileKey, parseSubject, weakFileKey } from '../subject/index.js' import { releaseIdentity, type ReleaseIdentity } from '../release/index.js' import { inferTotalParts, type YencMeta } from '../yenc/index.js' /** One segment, with the identity of the file it belongs to already resolved. */ export interface IngestSegment { fileKey: string poster: string /** Subject without segment counter. */ subject: string name: string totalParts: number number: number /** Without angle brackets. */ messageId: string bytes: number articleNumber?: bigint | number postedAt?: Date groups: string[] /** * The identity comes from the yEnc name only (obfuscated posts). Such names * can be shared by several equal-sized files of one post, so a second * article for the same part number marks the binary as conflicted. */ weakIdentity?: boolean /** Release (download) this file belongs to, when it can be determined. */ release?: ReleaseIdentity | null /** Exact file size from =ybegin (body-derived segments only). */ size?: number /** yEnc CRCs (body-derived segments only), to verify split volumes. */ pcrc32?: string partSize?: number fileCrc32?: string /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */ md5_16k?: string } /** An XOVER overview entry as returned by nntp-js. */ export type OverviewHeader = Record function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) { const xrefGroups = parseXrefGroups(header.xref) return { poster: header.from ?? '', messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0, articleNumber, postedAt: header.date ? new Date(header.date) : undefined, groups: xrefGroups.length > 0 ? xrefGroups : [groupName], } } /** Builds a segment from the subject line; null when the subject is obfuscated. */ export function segmentFromHeader( header: OverviewHeader, groupName: string, articleNumber?: bigint | number ): IngestSegment | null { const parsed = header.subject ? parseSubject(header.subject) : null if (!parsed || !header['message-id']) return null const base = common(header, groupName, articleNumber) return { ...base, fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments), subject: parsed.base, name: parsed.name, totalParts: parsed.totalSegments, number: parsed.segment, release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }), } } /** * Builds a segment for an obfuscated post from the yEnc header in its body. * Subject and poster can be random per article, so the identity comes from * the yEnc header only (see weakFileKey). */ export function segmentFromYenc( header: OverviewHeader, groupName: string, meta: Pick & Partial>, articleNumber?: bigint | number ): IngestSegment | null { const base = common(header, groupName, articleNumber) const totalParts = inferTotalParts(meta) const number = meta.begin.part ?? 1 if (!base.messageId || number < 1 || number > totalParts) return null const subject = `"${meta.begin.name}" yEnc` return { ...base, fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts), subject, name: meta.begin.name, totalParts, number, weakIdentity: true, release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }), size: meta.begin.size, // Single-part posts only carry crc32, which is then also the part's CRC. pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32), partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size, fileCrc32: meta.end?.crc32, } }