Rebuild: a file's identity is computed once per batch, not per article
The file key and release identity (two SHA-1 hashes and the release rules) were ~25% of the edge worker's time. edge:bench: boneless 100k → 137k headers/s on one core, ath (2013) 70k → 143k. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cd06f96f85
commit
981bc01810
3 files changed
+26
-6
No files matched your search
+20
-3
@@ -51,22 +51,39 @@ function common(header: OverviewHeader, groupName: string, articleNumber?: bigin
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A file's identity, the same for all of its articles: worth computing once
|
||||
* per file (two SHA-1 hashes and the release rules were ~25% of an edge
|
||||
* worker's time; LOG.md 2026-10-09).
|
||||
*/
|
||||
export type IdentityCache = Map<string, { fileKey: string; release: ReleaseIdentity | null }>
|
||||
|
||||
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
||||
export function segmentFromHeader(
|
||||
header: OverviewHeader,
|
||||
groupName: string,
|
||||
articleNumber?: bigint | number
|
||||
articleNumber?: bigint | number,
|
||||
cache?: IdentityCache
|
||||
): IngestSegment | null {
|
||||
const parsed = header.subject ? parseSubject(header.subject) : null
|
||||
if (!parsed || !header['message-id']) return null
|
||||
const base = common(header, groupName, articleNumber)
|
||||
const id = `${base.poster}\n${parsed.base}\n${parsed.totalSegments}`
|
||||
let identity = cache?.get(id)
|
||||
if (!identity) {
|
||||
identity = {
|
||||
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
||||
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
||||
}
|
||||
cache?.set(id, identity)
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
||||
fileKey: identity.fileKey,
|
||||
subject: parsed.base,
|
||||
name: parsed.name,
|
||||
totalParts: parsed.totalSegments,
|
||||
number: parsed.segment,
|
||||
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
||||
release: identity.release,
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user