Rebuild: a file's identity is computed once per batch, not per article
The file key and release identity (two SHA-1 hashes and the release rules) were ~25% of the edge worker's time. edge:bench: boneless 100k → 137k headers/s on one core, ath (2013) 70k → 143k. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cd06f96f85
commit
981bc01810
3 files changed
+26
-6
No files matched your search
@@ -1,7 +1,7 @@
|
|||||||
import { createHash } from 'node:crypto'
|
import { createHash } from 'node:crypto'
|
||||||
import type { Overview } from '../nntp/index.js'
|
import type { Overview } from '../nntp/index.js'
|
||||||
import { parseXrefGroups } from '../nntp/index.js'
|
import { parseXrefGroups } from '../nntp/index.js'
|
||||||
import { segmentFromHeader, type IngestSegment } from '../headers/index.js'
|
import { segmentFromHeader, type IdentityCache, type IngestSegment } from '../headers/index.js'
|
||||||
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
||||||
import { RuleSet, isRandomFile, ownerOf, type Rules } from '../rules/index.js'
|
import { RuleSet, isRandomFile, ownerOf, type Rules } from '../rules/index.js'
|
||||||
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
|
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
|
||||||
@@ -104,6 +104,8 @@ export class RangeIndexer {
|
|||||||
private rules: RuleSet
|
private rules: RuleSet
|
||||||
private batch: EdgeBatch
|
private batch: EdgeBatch
|
||||||
private counts = new Map<string, RuleCount>()
|
private counts = new Map<string, RuleCount>()
|
||||||
|
/** File identities seen in this batch (cleared with each take, so it stays small). */
|
||||||
|
private identities: IdentityCache = new Map()
|
||||||
private readonly idle: number
|
private readonly idle: number
|
||||||
private readonly edge: number
|
private readonly edge: number
|
||||||
|
|
||||||
@@ -127,7 +129,7 @@ export class RangeIndexer {
|
|||||||
this.count('crosspost', owner, 1, 0)
|
this.count('crosspost', owner, 1, 0)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
const segment = segmentFromHeader(header, group, articleNumber)
|
const segment = segmentFromHeader(header, group, articleNumber, this.identities)
|
||||||
if (segment) segments.push(segment)
|
if (segment) segments.push(segment)
|
||||||
else this.count('obfuscated', '', 1, 0)
|
else this.count('obfuscated', '', 1, 0)
|
||||||
}
|
}
|
||||||
@@ -145,6 +147,7 @@ export class RangeIndexer {
|
|||||||
/** What was found since the last take, and a fresh batch. */
|
/** What was found since the last take, and a fresh batch. */
|
||||||
take(): EdgeBatch {
|
take(): EdgeBatch {
|
||||||
this.judgeReleases()
|
this.judgeReleases()
|
||||||
|
this.identities.clear()
|
||||||
const out = this.batch
|
const out = this.batch
|
||||||
out.counts = [...this.counts.values()]
|
out.counts = [...this.counts.values()]
|
||||||
this.counts.clear()
|
this.counts.clear()
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
export { parseSubject, type ParsedSubject } from './subject.js'
|
export { parseSubject, type ParsedSubject } from './subject.js'
|
||||||
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
|
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
|
||||||
export { segmentFromHeader, type IngestSegment, type OverviewHeader } from './segment.js'
|
export { segmentFromHeader, type IngestSegment, type OverviewHeader, type IdentityCache } from './segment.js'
|
||||||
+20
-3
@@ -51,22 +51,39 @@ function common(header: OverviewHeader, groupName: string, articleNumber?: bigin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A file's identity, the same for all of its articles: worth computing once
|
||||||
|
* per file (two SHA-1 hashes and the release rules were ~25% of an edge
|
||||||
|
* worker's time; LOG.md 2026-10-09).
|
||||||
|
*/
|
||||||
|
export type IdentityCache = Map<string, { fileKey: string; release: ReleaseIdentity | null }>
|
||||||
|
|
||||||
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
||||||
export function segmentFromHeader(
|
export function segmentFromHeader(
|
||||||
header: OverviewHeader,
|
header: OverviewHeader,
|
||||||
groupName: string,
|
groupName: string,
|
||||||
articleNumber?: bigint | number
|
articleNumber?: bigint | number,
|
||||||
|
cache?: IdentityCache
|
||||||
): IngestSegment | null {
|
): IngestSegment | null {
|
||||||
const parsed = header.subject ? parseSubject(header.subject) : null
|
const parsed = header.subject ? parseSubject(header.subject) : null
|
||||||
if (!parsed || !header['message-id']) return null
|
if (!parsed || !header['message-id']) return null
|
||||||
const base = common(header, groupName, articleNumber)
|
const base = common(header, groupName, articleNumber)
|
||||||
|
const id = `${base.poster}\n${parsed.base}\n${parsed.totalSegments}`
|
||||||
|
let identity = cache?.get(id)
|
||||||
|
if (!identity) {
|
||||||
|
identity = {
|
||||||
|
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
||||||
|
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
||||||
|
}
|
||||||
|
cache?.set(id, identity)
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
...base,
|
...base,
|
||||||
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
fileKey: identity.fileKey,
|
||||||
subject: parsed.base,
|
subject: parsed.base,
|
||||||
name: parsed.name,
|
name: parsed.name,
|
||||||
totalParts: parsed.totalSegments,
|
totalParts: parsed.totalSegments,
|
||||||
number: parsed.segment,
|
number: parsed.segment,
|
||||||
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
release: identity.release,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user