Daan MeijerandClaude Opus 5.5
d773c91ad0
lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys, segments from overview lines; the yEnc path stays behind for phase 2), lib/release (identity, random names, now in lib), lib/files (aggregator, volume names), lib/rules (posting tools, owner by priority, release verdicts), lib/chunks (release chunk codec). Their unit tests came along; new ones for codec, rules and volumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
73 lines
2.5 KiB
TypeScript
73 lines
2.5 KiB
TypeScript
import { parseXrefGroups } from '../nntp/xref.js'
|
|
import { fileKey } from './file_key.js'
|
|
import { parseSubject } from './subject.js'
|
|
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
|
|
|
|
/** One segment, with the identity of the file it belongs to already resolved. */
|
|
export interface IngestSegment {
|
|
fileKey: string
|
|
poster: string
|
|
/** Subject without segment counter. */
|
|
subject: string
|
|
name: string
|
|
totalParts: number
|
|
number: number
|
|
/** Without angle brackets. */
|
|
messageId: string
|
|
bytes: number
|
|
articleNumber?: bigint | number
|
|
postedAt?: Date
|
|
groups: string[]
|
|
/**
|
|
* The identity comes from the yEnc name only (obfuscated posts). Such names
|
|
* can be shared by several equal-sized files of one post, so a second
|
|
* article for the same part number marks the binary as conflicted.
|
|
*/
|
|
weakIdentity?: boolean
|
|
/** Release (download) this file belongs to, when it can be determined. */
|
|
release?: ReleaseIdentity | null
|
|
/** Exact file size from =ybegin (body-derived segments only). */
|
|
size?: number
|
|
/** yEnc CRCs (body-derived segments only), to verify split volumes. */
|
|
pcrc32?: string
|
|
partSize?: number
|
|
fileCrc32?: string
|
|
/** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
|
|
md5_16k?: string
|
|
}
|
|
|
|
/** An XOVER overview entry as returned by nntp-js. */
|
|
export type OverviewHeader = Record<string, string>
|
|
|
|
function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
|
|
const xrefGroups = parseXrefGroups(header.xref)
|
|
return {
|
|
poster: header.from ?? '',
|
|
messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
|
|
bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
|
|
articleNumber,
|
|
postedAt: header.date ? new Date(header.date) : undefined,
|
|
groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
|
|
}
|
|
}
|
|
|
|
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
|
export function segmentFromHeader(
|
|
header: OverviewHeader,
|
|
groupName: string,
|
|
articleNumber?: bigint | number
|
|
): IngestSegment | null {
|
|
const parsed = header.subject ? parseSubject(header.subject) : null
|
|
if (!parsed || !header['message-id']) return null
|
|
const base = common(header, groupName, articleNumber)
|
|
return {
|
|
...base,
|
|
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
|
subject: parsed.base,
|
|
name: parsed.name,
|
|
totalParts: parsed.totalSegments,
|
|
number: parsed.segment,
|
|
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
|
}
|
|
}
|