Rebuild: phase 1 modules from the current app, ordered by pipeline stage
lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys, segments from overview lines; the yEnc path stays behind for phase 2), lib/release (identity, random names, now in lib), lib/files (aggregator, volume names), lib/rules (posting tools, owner by priority, release verdicts), lib/chunks (release chunk codec). Their unit tests came along; new ones for codec, rules and volumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
96f0ade10e
commit
d773c91ad0
26 files changed
+1486
No files matched your search
@@ -0,0 +1,72 @@
|
||||
import { parseXrefGroups } from '../nntp/xref.js'
|
||||
import { fileKey } from './file_key.js'
|
||||
import { parseSubject } from './subject.js'
|
||||
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
|
||||
|
||||
/** One segment, with the identity of the file it belongs to already resolved. */
|
||||
export interface IngestSegment {
|
||||
fileKey: string
|
||||
poster: string
|
||||
/** Subject without segment counter. */
|
||||
subject: string
|
||||
name: string
|
||||
totalParts: number
|
||||
number: number
|
||||
/** Without angle brackets. */
|
||||
messageId: string
|
||||
bytes: number
|
||||
articleNumber?: bigint | number
|
||||
postedAt?: Date
|
||||
groups: string[]
|
||||
/**
|
||||
* The identity comes from the yEnc name only (obfuscated posts). Such names
|
||||
* can be shared by several equal-sized files of one post, so a second
|
||||
* article for the same part number marks the binary as conflicted.
|
||||
*/
|
||||
weakIdentity?: boolean
|
||||
/** Release (download) this file belongs to, when it can be determined. */
|
||||
release?: ReleaseIdentity | null
|
||||
/** Exact file size from =ybegin (body-derived segments only). */
|
||||
size?: number
|
||||
/** yEnc CRCs (body-derived segments only), to verify split volumes. */
|
||||
pcrc32?: string
|
||||
partSize?: number
|
||||
fileCrc32?: string
|
||||
/** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
|
||||
md5_16k?: string
|
||||
}
|
||||
|
||||
/** An XOVER overview entry as returned by nntp-js. */
|
||||
export type OverviewHeader = Record<string, string>
|
||||
|
||||
function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
|
||||
const xrefGroups = parseXrefGroups(header.xref)
|
||||
return {
|
||||
poster: header.from ?? '',
|
||||
messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
|
||||
bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
|
||||
articleNumber,
|
||||
postedAt: header.date ? new Date(header.date) : undefined,
|
||||
groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
|
||||
}
|
||||
}
|
||||
|
||||
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
||||
export function segmentFromHeader(
|
||||
header: OverviewHeader,
|
||||
groupName: string,
|
||||
articleNumber?: bigint | number
|
||||
): IngestSegment | null {
|
||||
const parsed = header.subject ? parseSubject(header.subject) : null
|
||||
if (!parsed || !header['message-id']) return null
|
||||
const base = common(header, groupName, articleNumber)
|
||||
return {
|
||||
...base,
|
||||
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
||||
subject: parsed.base,
|
||||
name: parsed.name,
|
||||
totalParts: parsed.totalSegments,
|
||||
number: parsed.segment,
|
||||
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user