Rebuild: a file's identity is computed once per batch, not per article

The file key and release identity (two SHA-1 hashes and the release rules)
were ~25% of the edge worker's time. edge:bench: boneless 100k → 137k
headers/s on one core, ath (2013) 70k → 143k.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 01:08:58 +02:00
1 parent cd06f96f85
commit 981bc01810
3 files changed
+26 -6

No files matched your search

+5 -2
View File
@@ -1,7 +1,7 @@
import { createHash } from 'node:crypto'
import type { Overview } from '../nntp/index.js'
import { parseXrefGroups } from '../nntp/index.js'
import { segmentFromHeader, type IngestSegment } from '../headers/index.js'
import { segmentFromHeader, type IdentityCache, type IngestSegment } from '../headers/index.js'
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
import { RuleSet, isRandomFile, ownerOf, type Rules } from '../rules/index.js'
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
@@ -104,6 +104,8 @@ export class RangeIndexer {
private rules: RuleSet
private batch: EdgeBatch
private counts = new Map<string, RuleCount>()
/** File identities seen in this batch (cleared with each take, so it stays small). */
private identities: IdentityCache = new Map()
private readonly idle: number
private readonly edge: number
@@ -127,7 +129,7 @@ export class RangeIndexer {
this.count('crosspost', owner, 1, 0)
continue
}
const segment = segmentFromHeader(header, group, articleNumber)
const segment = segmentFromHeader(header, group, articleNumber, this.identities)
if (segment) segments.push(segment)
else this.count('obfuscated', '', 1, 0)
}
@@ -145,6 +147,7 @@ export class RangeIndexer {
/** What was found since the last take, and a fresh batch. */
take(): EdgeBatch {
this.judgeReleases()
this.identities.clear()
const out = this.batch
out.counts = [...this.counts.values()]
this.counts.clear()
+1 -1
View File
@@ -1,3 +1,3 @@
export { parseSubject, type ParsedSubject } from './subject.js'
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
export { segmentFromHeader, type IngestSegment, type OverviewHeader } from './segment.js'
export { segmentFromHeader, type IngestSegment, type OverviewHeader, type IdentityCache } from './segment.js'
+20 -3
View File
@@ -51,22 +51,39 @@ function common(header: OverviewHeader, groupName: string, articleNumber?: bigin
}
}
/**
* A file's identity, the same for all of its articles: worth computing once
* per file (two SHA-1 hashes and the release rules were ~25% of an edge
* worker's time; LOG.md 2026-10-09).
*/
export type IdentityCache = Map<string, { fileKey: string; release: ReleaseIdentity | null }>
/** Builds a segment from the subject line; null when the subject is obfuscated. */
export function segmentFromHeader(
header: OverviewHeader,
groupName: string,
articleNumber?: bigint | number
articleNumber?: bigint | number,
cache?: IdentityCache
): IngestSegment | null {
const parsed = header.subject ? parseSubject(header.subject) : null
if (!parsed || !header['message-id']) return null
const base = common(header, groupName, articleNumber)
const id = `${base.poster}\n${parsed.base}\n${parsed.totalSegments}`
let identity = cache?.get(id)
if (!identity) {
identity = {
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
}
cache?.set(id, identity)
}
return {
...base,
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
fileKey: identity.fileKey,
subject: parsed.base,
name: parsed.name,
totalParts: parsed.totalSegments,
number: parsed.segment,
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
release: identity.release,
}
}