Rebuild: strings that outlive their chunk are copied (worker 1.14 → 0.66 GB)

A slice of a chunk's text keeps the whole text alive (tens of MB) for as
long as the file it belongs to stays open: message-ids, file details and
cache keys are now copies. edge:bench on boneless: peak RSS 1,137 → ~660
MB, external memory 570 → 40–75 MB, the same speed (~190k headers/s);
the same results on 800k real lines. edge:bench prints the breakdown.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/edge_bench.ts

@@ -97,6 +97,7 @@ export default class EdgeBench extends BaseCommand {
     let compressed = 0
     let text = 0
     let peakRss = 0
+    let peakDetail = ''
     const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
     const counts = new Map<string, number>()
     const keys = new Set<string>()
@@ -156,14 +157,18 @@ export default class EdgeBench extends BaseCommand {
       out.fragments += batch.fragments.length
       out.samples += batch.samples.length
       for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles)
-      peakRss = Math.max(peakRss, process.memoryUsage().rss)
+      const m = process.memoryUsage()
+      if (m.rss > peakRss) {
+        peakRss = m.rss
+        peakDetail = `heap ${(m.heapUsed / 1e6).toFixed(0)}/${(m.heapTotal / 1e6).toFixed(0)} MB, external ${(m.external / 1e6).toFixed(0)} MB, array buffers ${(m.arrayBuffers / 1e6).toFixed(0)} MB`
+      }
     }
     const cpu = process.cpuUsage(cpuStart)
     const cpuS = (cpu.user + cpu.system) / 1e6
     const total = t.inflate + t.parse + t.index
     this.logger.info(`${group} ${start}–${end}: ${files.length} chunks, ${(compressed / 1e6).toFixed(0)} MB compressed, ${(text / 1e6).toFixed(0)} MB text`)
     this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
-    this.logger.info(`  inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB`)
+    this.logger.info(`  inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`)
     this.logger.info(`  ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
     this.logger.info(`  kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
     for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(`  dropped ${k}: ${v.toLocaleString('en')} articles`)

lib/edge/scan.ts

@@ -26,6 +26,16 @@ export interface EdgeSegment {
   file: FileMeta
 }
 
+/**
+ * A copy of a string that doesn't keep the string it was cut from alive: a
+ * slice of a chunk's text otherwise holds on to the whole text (tens of MB)
+ * for as long as the file it belongs to stays open. External memory was
+ * ~570 MB of a worker's 1.1 GB (LOG.md 2026-10-09).
+ */
+function own(text: string): string {
+  return (' ' + text).slice(1)
+}
+
 interface Counter {
   index: number
   length: number
@@ -110,14 +120,14 @@ export class SegmentScanner {
     let file = this.files.get(key)
     if (file === undefined) {
       file = this.describe(subject, nameEnd, counter, from, xref, group)
-      this.files.set(key, file)
+      this.files.set(own(key), file)
     }
     if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null
     return {
       fileKey: file.fileKey,
       number: counter.segment,
       totalParts: file.totalParts,
-      messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId,
+      messageId: own(messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId),
       bytes: Number.parseInt(bytes, 10) || 0,
       articleNumber,
       postedAt: Date.parse(date),
@@ -129,17 +139,18 @@ export class SegmentScanner {
   private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
     const nameStart = subject.lastIndexOf('"', nameEnd - 1)
     if (nameStart === -1) return null
-    const name = subject.slice(nameStart + 1, nameEnd)
+    const name = own(subject.slice(nameStart + 1, nameEnd))
     if (!name || counter.total < 1) return null
-    const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim()
+    const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim())
+    const poster = own(from)
     return {
-      fileKey: fileKey(from, base, counter.total),
+      fileKey: fileKey(poster, base, counter.total),
       name,
-      poster: from,
+      poster,
       subject: base,
       totalParts: counter.total,
       groups: xrefGroups(xref, group),
-      release: releaseIdentity({ poster: from, subject: base, name, weak: false }),
+      release: releaseIdentity({ poster, subject: base, name, weak: false }),
     }
   }
 }
@@ -149,7 +160,7 @@ function xrefGroups(xref: string, group: string): string[] {
   const tokens = xref.trim().split(/\s+/)
   for (let i = 1; i < tokens.length; i++) {
     const colon = tokens[i].lastIndexOf(':')
-    if (colon > 0) groups.push(tokens[i].slice(0, colon))
+    if (colon > 0) groups.push(own(tokens[i].slice(0, colon)))
   }
   return groups.length > 0 ? groups : [group]
 }