Rebuild: a lean overview scan for the edge worker

The worker reads the XOVER reply's text directly: fields by position, the
owner from the Xref without arrays (crossposts leave before any subject
parsing), a subject parsed once per file (cached without its segment
counter), a file's details as one shared object, dates as numbers.
Garbage collection had been ~22% of the time.

Same results as parseOverviews + segmentFromHeader on 800k real lines.
edge:bench (one core): boneless 137k → 193k headers/s (CPU 21 → 12.8 s),
ath (2013) 143k → 303k; peak RSS 1.15 / 0.87 GB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/edge/range_indexer.ts

@@ -1,9 +1,8 @@
 import { createHash } from 'node:crypto'
 import type { Overview } from '../nntp/index.js'
-import { parseXrefGroups } from '../nntp/index.js'
-import { segmentFromHeader, type IdentityCache, type IngestSegment } from '../headers/index.js'
 import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
-import { RuleSet, isRandomFile, ownerOf, type Rules } from '../rules/index.js'
+import { RuleSet, isRandomFile, type Rules } from '../rules/index.js'
+import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
 import { encodeChunk, type ChunkFile } from '../chunks/index.js'
 
 /** A release's files from one batch (REBUILD.md, chunk store): what the catalog adds up, and the encoded chunk. */
@@ -100,12 +99,12 @@ const MAX_CANDIDATES = 4
  * waiting for parts.
  */
 export class RangeIndexer {
-  private aggregator = new FileAggregator<IngestSegment>()
+  private aggregator = new FileAggregator<EdgeSegment>()
+  private scanner = new SegmentScanner()
+  private segments: EdgeSegment[] = []
   private rules: RuleSet
   private batch: EdgeBatch
   private counts = new Map<string, RuleCount>()
-  /** File identities seen in this batch (cleared with each take, so it stays small). */
-  private identities: IdentityCache = new Map()
   private readonly idle: number
   private readonly edge: number
 
@@ -116,24 +115,38 @@ export class RangeIndexer {
     this.batch = this.empty(options.start)
   }
 
-  /** One chunk of overview lines; `next` is the first article after it. */
+  /** One chunk of overview lines (parsed); `next` is the first article after it. */
   add(overviews: Overview[], next: number) {
+    for (const [n, h] of overviews) this.line(n, h.subject ?? '', h.from ?? '', h.date ?? '', h['message-id'] ?? '', h[':bytes'] ?? '', h.xref ?? '')
+    this.endChunk(next)
+  }
+
+  /** One chunk as the XOVER reply's text (the worker's path: no object per line). */
+  addText(text: string, next: number) {
+    scanOverviewText(text, this.handleLine)
+    this.endChunk(next)
+  }
+
+  private handleLine = (n: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string) =>
+    this.line(n, subject, from, date, messageId, bytes, xref)
+
+  private line(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string) {
+    if (!Number.isFinite(articleNumber) || !messageId) return
     const { group, rules } = this.options
-    const segments: IngestSegment[] = []
-    for (const [articleNumber, header] of overviews) {
-      if (!Number.isFinite(articleNumber) || !header['message-id']) continue
-      this.batch.headers++
-      const groups = parseXrefGroups(header.xref)
-      const owner = ownerOf(groups.length > 0 ? groups : [group], rules.priorities) ?? group
-      if (owner !== group) {
-        this.count('crosspost', owner, 1, 0)
-        continue
-      }
-      const segment = segmentFromHeader(header, group, articleNumber, this.identities)
-      if (segment) segments.push(segment)
-      else this.count('obfuscated', '', 1, 0)
+    this.batch.headers++
+    const owner = xref ? this.scanner.owner(xref, group, rules.priorities) : group
+    if (owner !== group) {
+      this.count('crosspost', owner, 1, 0)
+      return
     }
-    this.files(this.aggregator.add(segments))
+    const segment = this.scanner.segment(articleNumber, subject, from, date, messageId, bytes, xref, group)
+    if (segment) this.segments.push(segment)
+    else this.count('obfuscated', '', 1, 0)
+  }
+
+  private endChunk(next: number) {
+    this.files(this.aggregator.add(this.segments))
+    this.segments = []
     this.cutOff(this.aggregator.evict(this.idle))
     this.batch.next = next
   }
@@ -147,7 +160,7 @@ export class RangeIndexer {
   /** What was found since the last take, and a fresh batch. */
   take(): EdgeBatch {
     this.judgeReleases()
-    this.identities.clear()
+    this.scanner.clear()
     const out = this.batch
     out.counts = [...this.counts.values()]
     this.counts.clear()
@@ -160,9 +173,9 @@ export class RangeIndexer {
   }
 
   /** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */
-  private completed = new Map<string, AggregatedFile<IngestSegment>[]>()
+  private completed = new Map<string, AggregatedFile<EdgeSegment>[]>()
 
-  private files(files: AggregatedFile<IngestSegment>[]) {
+  private files(files: AggregatedFile<EdgeSegment>[]) {
     for (const file of files) {
       const key = releaseKeyOf(file.segments[0])
       const list = this.completed.get(key)
@@ -178,7 +191,7 @@ export class RangeIndexer {
     for (const [key, list] of byRelease) {
       const verdict = this.rules.judge(
         key,
-        list.map((f) => ({ name: f.segments[0].name, totalParts: f.segments[0].totalParts, groups: f.segments[0].groups, firstMessageId: f.segments[0].messageId }))
+        list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId }))
       )
       if (verdict.rule === null) {
         this.batch.releases.push(chunkOf(key, list))
@@ -191,7 +204,7 @@ export class RangeIndexer {
   }
 
   /** Given-up segments: kept per file near the range's edges (the neighbour may complete them), counted otherwise. */
-  private cutOff(segments: IngestSegment[]) {
+  private cutOff(segments: EdgeSegment[]) {
     const { start, end } = this.options
     const byFile = new Map<string, Fragment>()
     for (const s of segments) {
@@ -207,8 +220,8 @@ export class RangeIndexer {
         this.batch.fragments.push(fragment)
       }
       fragment.segments.push([s.number, s.messageId, s.bytes, n])
-      const t = s.postedAt?.getTime()
-      if (t !== undefined && Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t
+      const t = s.postedAt
+      if (Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t
     }
   }
 
@@ -229,24 +242,24 @@ export class RangeIndexer {
 }
 
 /** The release key; a file without a recognisable release becomes a release of its own. */
-function releaseKeyOf(s: IngestSegment): string {
-  return s.release?.key ?? createHash('sha1').update(`file\n${s.fileKey}`).digest('hex')
+function releaseKeyOf(s: EdgeSegment): string {
+  return s.file.release?.key ?? createHash('sha1').update(`file\n${s.fileKey}`).digest('hex')
 }
 
-function postedOf(segments: IngestSegment[]): number | null {
+function postedOf(segments: EdgeSegment[]): number | null {
   let min: number | null = null
   for (const s of segments) {
-    const t = s.postedAt?.getTime()
-    if (t !== undefined && Number.isFinite(t) && (min === null || t < min)) min = t
+    const t = s.postedAt
+    if (Number.isFinite(t) && (min === null || t < min)) min = t
   }
   return min
 }
 
-function chunkOf(key: string, files: AggregatedFile<IngestSegment>[]): ReleaseChunk {
-  const first = files[0].segments[0]
+function chunkOf(key: string, files: AggregatedFile<EdgeSegment>[]): ReleaseChunk {
+  const first = files[0].segments[0].file
   const chunkFiles: ChunkFile[] = files.map((f) => {
-    const s = f.segments[0]
-    return { name: s.name, poster: s.poster, subject: s.subject, postedAt: postedOf(f.segments), groups: s.groups, segments: f.segments.map((x) => [x.number, x.messageId, x.bytes]) }
+    const m = f.segments[0].file
+    return { name: m.name, poster: m.poster, subject: m.subject, postedAt: postedOf(f.segments), groups: m.groups, segments: f.segments.map((x) => [x.number, x.messageId, x.bytes]) }
   })
   let segments = 0
   let bytes = 0
@@ -257,7 +270,7 @@ function chunkOf(key: string, files: AggregatedFile<IngestSegment>[]): ReleaseCh
     for (const s of f.segments) {
       segments++
       bytes += s.bytes
-      const n = Number(s.articleNumber)
+      const n = s.articleNumber
       if (n < firstArticle) firstArticle = n
       if (n > lastArticle) lastArticle = n
     }
@@ -268,7 +281,7 @@ function chunkOf(key: string, files: AggregatedFile<IngestSegment>[]): ReleaseCh
     key,
     name: first.release?.name ?? first.name,
     source: first.release?.source ?? 'file',
-    fileTotal: files.find((f) => f.segments[0].release?.fileTotal)?.segments[0].release?.fileTotal ?? null,
+    fileTotal: files.find((f) => f.segments[0].file.release?.fileTotal)?.segments[0].file.release?.fileTotal ?? null,
     poster: first.poster,
     groups: first.groups,
     postedAt,
@@ -288,11 +301,11 @@ function chunkOf(key: string, files: AggregatedFile<IngestSegment>[]): ReleaseCh
  * volume, of its par2 index, of a lone random-named file, and a 7z set's
  * last volume (its listing is in the tail). At most a few per release.
  */
-function candidatesOf(key: string, files: AggregatedFile<IngestSegment>[]): ProbeCandidate[] {
+function candidatesOf(key: string, files: AggregatedFile<EdgeSegment>[]): ProbeCandidate[] {
   const out: ProbeCandidate[] = []
-  let tail: { volume: number; file: AggregatedFile<IngestSegment> } | null = null
+  let tail: { volume: number; file: AggregatedFile<EdgeSegment> } | null = null
   for (const f of files) {
-    const name = f.segments[0].name
+    const name = f.segments[0].file.name
     const volume = sevenZipVolume(name)
     if (volume !== null && volume > 1) {
       if (!tail || volume > tail.volume) tail = { volume, file: f }
@@ -301,22 +314,23 @@ function candidatesOf(key: string, files: AggregatedFile<IngestSegment>[]): Prob
     if (isArchiveStart(name)) out.push({ releaseKey: key, kind: 'archive', messageId: f.segments[0].messageId, file: name })
     else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, kind: 'par2', messageId: f.segments[0].messageId, file: name })
   }
-  if (tail) out.push({ releaseKey: key, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].name })
-  if (out.length === 0 && files.length === 1 && isRandomFile(files[0].segments[0].name)) {
-    out.push({ releaseKey: key, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].name })
+  if (tail) out.push({ releaseKey: key, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].file.name })
+  if (out.length === 0 && files.length === 1 && isRandomFile(files[0].segments[0].file.name)) {
+    out.push({ releaseKey: key, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].file.name })
   }
   return out.slice(0, MAX_CANDIDATES)
 }
 
-function fragmentOf(s: IngestSegment): Fragment {
+function fragmentOf(s: EdgeSegment): Fragment {
+  const m = s.file
   return {
     fileKey: s.fileKey,
-    name: s.name,
-    poster: s.poster,
-    subject: s.subject,
-    totalParts: s.totalParts,
-    groups: s.groups,
-    release: s.release ? { key: s.release.key, name: s.release.name, source: s.release.source, ...(s.release.fileTotal ? { fileTotal: s.release.fileTotal } : {}) } : null,
+    name: m.name,
+    poster: m.poster,
+    subject: m.subject,
+    totalParts: m.totalParts,
+    groups: m.groups,
+    release: m.release ? { key: m.release.key, name: m.release.name, source: m.release.source, ...(m.release.fileTotal ? { fileTotal: m.release.fileTotal } : {}) } : null,
     postedAt: null,
     segments: [],
   }

lib/edge/scan.ts

@@ -0,0 +1,181 @@
+import { fileKey } from '../headers/index.js'
+import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
+
+/** A file's details, shared by all of its segments (one object per file, not per article). */
+export interface FileMeta {
+  fileKey: string
+  name: string
+  poster: string
+  /** The subject without its segment counter. */
+  subject: string
+  totalParts: number
+  groups: string[]
+  release: ReleaseIdentity | null
+}
+
+/** One article of a file, as lean as the edge worker can keep it. */
+export interface EdgeSegment {
+  fileKey: string
+  number: number
+  totalParts: number
+  messageId: string
+  bytes: number
+  articleNumber: number
+  /** ms since 1970, NaN when the Date header doesn't parse. */
+  postedAt: number
+  file: FileMeta
+}
+
+interface ParsedSubjectBase {
+  name: string
+  totalSegments: number
+  base: string
+}
+
+const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
+
+/**
+ * Builds segments from overview fields (REBUILD.md, edge worker), the same
+ * as lib/headers' segmentFromHeader but allocating as little as it can: a
+ * subject is parsed once per file (cached without its segment counter), a
+ * file's details are one shared object, and the Xref is read without
+ * building arrays. Garbage collection was ~20% of the worker's time
+ * (LOG.md 2026-10-09). The caches live until `clear`, once per batch.
+ */
+export class SegmentScanner {
+  private subjects = new Map<string, ParsedSubjectBase | null>()
+  private files = new Map<string, FileMeta>()
+
+  clear() {
+    this.subjects.clear()
+    this.files.clear()
+  }
+
+  /** The owner of an article by its Xref field: the indexed group with the lowest priority, else `group`. */
+  owner(xref: string, group: string, priorities: Record<string, number>): string {
+    // "news.example a.b.test:123 a.b.x:456": the first token is the server.
+    let at = xref.indexOf(' ')
+    if (at === -1) return group
+    let owner: string | null = null
+    let best = Number.POSITIVE_INFINITY
+    while (at !== -1) {
+      while (xref.charCodeAt(at) === 32) at++
+      if (at >= xref.length) break
+      const end = xref.indexOf(' ', at)
+      const colon = xref.lastIndexOf(':', end === -1 ? xref.length : end)
+      if (colon > at) {
+        const name = xref.slice(at, colon)
+        const p = priorities[name]
+        if (p !== undefined && p < best) {
+          best = p
+          owner = name
+        }
+      }
+      at = end
+    }
+    return owner ?? group
+  }
+
+  /** A segment from an article's fields, or null when its subject is obfuscated. */
+  segment(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string, group: string): EdgeSegment | null {
+    if (!subject || !messageId) return null
+    const nameEnd = subject.lastIndexOf('"')
+    if (nameEnd <= 0) return null
+    // The segment counter: the last "(k/N)" after the filename (a file size may follow it).
+    SEGMENT_COUNTER.lastIndex = nameEnd
+    let counter: RegExpExecArray | null = null
+    for (let m = SEGMENT_COUNTER.exec(subject); m; m = SEGMENT_COUNTER.exec(subject)) counter = m
+    if (!counter) return null
+    const key = subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter[0].length)
+    let parsed = this.subjects.get(key)
+    if (parsed === undefined) {
+      parsed = parseBase(subject, nameEnd, counter)
+      this.subjects.set(key, parsed)
+    }
+    if (!parsed) return null
+    const number = Number.parseInt(counter[1], 10)
+    if (number < 1 || number > parsed.totalSegments) return null
+    const id = `${from}\n${parsed.base}\n${parsed.totalSegments}`
+    let file = this.files.get(id)
+    if (!file) {
+      file = {
+        fileKey: fileKey(from, parsed.base, parsed.totalSegments),
+        name: parsed.name,
+        poster: from,
+        subject: parsed.base,
+        totalParts: parsed.totalSegments,
+        groups: xrefGroups(xref, group),
+        release: releaseIdentity({ poster: from, subject: parsed.base, name: parsed.name, weak: false }),
+      }
+      this.files.set(id, file)
+    }
+    return {
+      fileKey: file.fileKey,
+      number,
+      totalParts: file.totalParts,
+      messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId,
+      bytes: Number.parseInt(bytes, 10) || 0,
+      articleNumber,
+      postedAt: Date.parse(date),
+      file,
+    }
+  }
+}
+
+/** As parseSubject: the name, the segment count and the base, from a subject whose counter was found. */
+function parseBase(subject: string, nameEnd: number, counter: RegExpExecArray): ParsedSubjectBase | null {
+  const nameStart = subject.lastIndexOf('"', nameEnd - 1)
+  if (nameStart === -1) return null
+  const name = subject.slice(nameStart + 1, nameEnd)
+  if (!name) return null
+  const totalSegments = Number.parseInt(counter[2], 10)
+  if (totalSegments < 1) return null
+  const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length)).replace(/\s+/g, ' ').trim()
+  return { name, totalSegments, base }
+}
+
+function xrefGroups(xref: string, group: string): string[] {
+  const groups: string[] = []
+  const tokens = xref.trim().split(/\s+/)
+  for (let i = 1; i < tokens.length; i++) {
+    const colon = tokens[i].lastIndexOf(':')
+    if (colon > 0) groups.push(tokens[i].slice(0, colon))
+  }
+  return groups.length > 0 ? groups : [group]
+}
+
+/**
+ * Calls `line` for each overview line of an XOVER reply's text, with its
+ * fields as strings (no object per line): number, subject, from, date,
+ * message-id, :bytes, and the Xref without its "Xref: " name.
+ */
+export function scanOverviewText(
+  text: string,
+  line: (number: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string) => void
+) {
+  let start = 0
+  const length = text.length
+  while (start < length) {
+    let end = text.indexOf('\n', start)
+    if (end === -1) end = length
+    const stop = end > start && text.charCodeAt(end - 1) === 13 ? end - 1 : end
+    if (stop > start) {
+      // number, subject, from, date, message-id, references, bytes, lines, xref
+      const t1 = text.indexOf('\t', start)
+      const t2 = t1 === -1 ? -1 : text.indexOf('\t', t1 + 1)
+      const t3 = t2 === -1 ? -1 : text.indexOf('\t', t2 + 1)
+      const t4 = t3 === -1 ? -1 : text.indexOf('\t', t3 + 1)
+      const t5 = t4 === -1 ? -1 : text.indexOf('\t', t4 + 1)
+      const t6 = t5 === -1 ? -1 : text.indexOf('\t', t5 + 1)
+      const t7 = t6 === -1 ? -1 : text.indexOf('\t', t6 + 1)
+      const t8 = t7 === -1 ? -1 : text.indexOf('\t', t7 + 1)
+      if (t5 !== -1 && t5 < stop) {
+        const field = (a: number, b: number) => (a === -1 || a >= stop ? '' : text.slice(a + 1, b === -1 || b > stop ? stop : b))
+        let xref = t8 === -1 || t8 >= stop ? '' : field(t8, text.indexOf('\t', t8 + 1))
+        if (xref.startsWith('Xref:')) xref = xref.slice(xref.charCodeAt(5) === 32 ? 6 : 5)
+        line(Number.parseInt(text.slice(start, t1), 10), text.slice(t1 + 1, t2), field(t2, t3), field(t3, t4), field(t4, t5), field(t6, t7), xref)
+      }
+    }
+    start = end + 1
+  }
+}