Daan MeijerandClaude Opus 5.5
981bc01810
The file key and release identity (two SHA-1 hashes and the release rules) were ~25% of the edge worker's time. edge:bench: boneless 100k → 137k headers/s on one core, ath (2013) 70k → 143k. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
90 lines
3.1 KiB
TypeScript
90 lines
3.1 KiB
TypeScript
import { parseXrefGroups } from '../nntp/xref.js'
|
|
import { fileKey } from './file_key.js'
|
|
import { parseSubject } from './subject.js'
|
|
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
|
|
|
|
/** One segment, with the identity of the file it belongs to already resolved. */
|
|
export interface IngestSegment {
|
|
fileKey: string
|
|
poster: string
|
|
/** Subject without segment counter. */
|
|
subject: string
|
|
name: string
|
|
totalParts: number
|
|
number: number
|
|
/** Without angle brackets. */
|
|
messageId: string
|
|
bytes: number
|
|
articleNumber?: bigint | number
|
|
postedAt?: Date
|
|
groups: string[]
|
|
/**
|
|
* The identity comes from the yEnc name only (obfuscated posts). Such names
|
|
* can be shared by several equal-sized files of one post, so a second
|
|
* article for the same part number marks the binary as conflicted.
|
|
*/
|
|
weakIdentity?: boolean
|
|
/** Release (download) this file belongs to, when it can be determined. */
|
|
release?: ReleaseIdentity | null
|
|
/** Exact file size from =ybegin (body-derived segments only). */
|
|
size?: number
|
|
/** yEnc CRCs (body-derived segments only), to verify split volumes. */
|
|
pcrc32?: string
|
|
partSize?: number
|
|
fileCrc32?: string
|
|
/** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
|
|
md5_16k?: string
|
|
}
|
|
|
|
/** An XOVER overview entry as returned by nntp-js. */
|
|
export type OverviewHeader = Record<string, string>
|
|
|
|
function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
|
|
const xrefGroups = parseXrefGroups(header.xref)
|
|
return {
|
|
poster: header.from ?? '',
|
|
messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
|
|
bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
|
|
articleNumber,
|
|
postedAt: header.date ? new Date(header.date) : undefined,
|
|
groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A file's identity, the same for all of its articles: worth computing once
|
|
* per file (two SHA-1 hashes and the release rules were ~25% of an edge
|
|
* worker's time; LOG.md 2026-10-09).
|
|
*/
|
|
export type IdentityCache = Map<string, { fileKey: string; release: ReleaseIdentity | null }>
|
|
|
|
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
|
export function segmentFromHeader(
|
|
header: OverviewHeader,
|
|
groupName: string,
|
|
articleNumber?: bigint | number,
|
|
cache?: IdentityCache
|
|
): IngestSegment | null {
|
|
const parsed = header.subject ? parseSubject(header.subject) : null
|
|
if (!parsed || !header['message-id']) return null
|
|
const base = common(header, groupName, articleNumber)
|
|
const id = `${base.poster}\n${parsed.base}\n${parsed.totalSegments}`
|
|
let identity = cache?.get(id)
|
|
if (!identity) {
|
|
identity = {
|
|
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
|
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
|
}
|
|
cache?.set(id, identity)
|
|
}
|
|
return {
|
|
...base,
|
|
fileKey: identity.fileKey,
|
|
subject: parsed.base,
|
|
name: parsed.name,
|
|
totalParts: parsed.totalSegments,
|
|
number: parsed.segment,
|
|
release: identity.release,
|
|
}
|
|
}
|