Rebuild: a file's identity is computed once per batch, not per article
The file key and release identity (two SHA-1 hashes and the release rules) were ~25% of the edge worker's time. edge:bench: boneless 100k → 137k headers/s on one core, ath (2013) 70k → 143k. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cd06f96f85
commit
981bc01810
3 files changed
+26
-6
No files matched your search
@@ -1,7 +1,7 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import type { Overview } from '../nntp/index.js'
|
||||
import { parseXrefGroups } from '../nntp/index.js'
|
||||
import { segmentFromHeader, type IngestSegment } from '../headers/index.js'
|
||||
import { segmentFromHeader, type IdentityCache, type IngestSegment } from '../headers/index.js'
|
||||
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
||||
import { RuleSet, isRandomFile, ownerOf, type Rules } from '../rules/index.js'
|
||||
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
|
||||
@@ -104,6 +104,8 @@ export class RangeIndexer {
|
||||
private rules: RuleSet
|
||||
private batch: EdgeBatch
|
||||
private counts = new Map<string, RuleCount>()
|
||||
/** File identities seen in this batch (cleared with each take, so it stays small). */
|
||||
private identities: IdentityCache = new Map()
|
||||
private readonly idle: number
|
||||
private readonly edge: number
|
||||
|
||||
@@ -127,7 +129,7 @@ export class RangeIndexer {
|
||||
this.count('crosspost', owner, 1, 0)
|
||||
continue
|
||||
}
|
||||
const segment = segmentFromHeader(header, group, articleNumber)
|
||||
const segment = segmentFromHeader(header, group, articleNumber, this.identities)
|
||||
if (segment) segments.push(segment)
|
||||
else this.count('obfuscated', '', 1, 0)
|
||||
}
|
||||
@@ -145,6 +147,7 @@ export class RangeIndexer {
|
||||
/** What was found since the last take, and a fresh batch. */
|
||||
take(): EdgeBatch {
|
||||
this.judgeReleases()
|
||||
this.identities.clear()
|
||||
const out = this.batch
|
||||
out.counts = [...this.counts.values()]
|
||||
this.counts.clear()
|
||||
|
||||
Reference in new issue
Block a user