From 981bc018106cb1f1e3b7fac05397c8d351b51128 Mon Sep 17 00:00:00 2001
From: Daan Meijer <daan@studioseptember.nl>
Date: Fri, 9 Oct 2026 01:08:58 +0200
Subject: [PATCH] Rebuild: a file's identity is computed once per batch, not
 per article
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The file key and release identity (two SHA-1 hashes and the release rules)
were ~25% of the edge worker's time. edge:bench: boneless 100k → 137k
headers/s on one core, ath (2013) 70k → 143k.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 lib/edge/range_indexer.ts |  7 +++++--
 lib/headers/index.ts      |  2 +-
 lib/headers/segment.ts    | 23 ++++++++++++++++++++---
 3 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts
index 5c5c438..67cbf2f 100644
--- a/lib/edge/range_indexer.ts
+++ b/lib/edge/range_indexer.ts
@@ -1,7 +1,7 @@
 import { createHash } from 'node:crypto'
 import type { Overview } from '../nntp/index.js'
 import { parseXrefGroups } from '../nntp/index.js'
-import { segmentFromHeader, type IngestSegment } from '../headers/index.js'
+import { segmentFromHeader, type IdentityCache, type IngestSegment } from '../headers/index.js'
 import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
 import { RuleSet, isRandomFile, ownerOf, type Rules } from '../rules/index.js'
 import { encodeChunk, type ChunkFile } from '../chunks/index.js'
@@ -104,6 +104,8 @@ export class RangeIndexer {
   private rules: RuleSet
   private batch: EdgeBatch
   private counts = new Map<string, RuleCount>()
+  /** File identities seen in this batch (cleared with each take, so it stays small). */
+  private identities: IdentityCache = new Map()
   private readonly idle: number
   private readonly edge: number
 
@@ -127,7 +129,7 @@ export class RangeIndexer {
         this.count('crosspost', owner, 1, 0)
         continue
       }
-      const segment = segmentFromHeader(header, group, articleNumber)
+      const segment = segmentFromHeader(header, group, articleNumber, this.identities)
       if (segment) segments.push(segment)
       else this.count('obfuscated', '', 1, 0)
     }
@@ -145,6 +147,7 @@ export class RangeIndexer {
   /** What was found since the last take, and a fresh batch. */
   take(): EdgeBatch {
     this.judgeReleases()
+    this.identities.clear()
     const out = this.batch
     out.counts = [...this.counts.values()]
     this.counts.clear()
diff --git a/lib/headers/index.ts b/lib/headers/index.ts
index 64392ac..24639a6 100644
--- a/lib/headers/index.ts
+++ b/lib/headers/index.ts
@@ -1,3 +1,3 @@
 export { parseSubject, type ParsedSubject } from './subject.js'
 export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
-export { segmentFromHeader, type IngestSegment, type OverviewHeader } from './segment.js'
+export { segmentFromHeader, type IngestSegment, type OverviewHeader, type IdentityCache } from './segment.js'
diff --git a/lib/headers/segment.ts b/lib/headers/segment.ts
index db97f15..362ccc3 100644
--- a/lib/headers/segment.ts
+++ b/lib/headers/segment.ts
@@ -51,22 +51,39 @@ function common(header: OverviewHeader, groupName: string, articleNumber?: bigin
   }
 }
 
+/**
+ * A file's identity, the same for all of its articles: worth computing once
+ * per file (two SHA-1 hashes and the release rules were ~25% of an edge
+ * worker's time; LOG.md 2026-10-09).
+ */
+export type IdentityCache = Map<string, { fileKey: string; release: ReleaseIdentity | null }>
+
 /** Builds a segment from the subject line; null when the subject is obfuscated. */
 export function segmentFromHeader(
   header: OverviewHeader,
   groupName: string,
-  articleNumber?: bigint | number
+  articleNumber?: bigint | number,
+  cache?: IdentityCache
 ): IngestSegment | null {
   const parsed = header.subject ? parseSubject(header.subject) : null
   if (!parsed || !header['message-id']) return null
   const base = common(header, groupName, articleNumber)
+  const id = `${base.poster}\n${parsed.base}\n${parsed.totalSegments}`
+  let identity = cache?.get(id)
+  if (!identity) {
+    identity = {
+      fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
+      release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
+    }
+    cache?.set(id, identity)
+  }
   return {
     ...base,
-    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
+    fileKey: identity.fileKey,
     subject: parsed.base,
     name: parsed.name,
     totalParts: parsed.totalSegments,
     number: parsed.segment,
-    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
+    release: identity.release,
   }
 }