Rebuild: strings that outlive their chunk are copied (worker 1.14 → 0.66 GB)
Test and deploy / test (push) Successful in 18s
Test and deploy / deploy (push) Skipped

A slice of a chunk's text keeps the whole text alive (tens of MB) for as
long as the file it belongs to stays open: message-ids, file details and
cache keys are now copies. edge:bench on boneless: peak RSS 1,137 → ~660
MB, external memory 570 → 40–75 MB, the same speed (~190k headers/s);
the same results on 800k real lines. edge:bench prints the breakdown.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 02:36:34 +02:00
1 parent 80c920f678
commit 73fc9aabdf
2 files changed
+26 -10

No files matched your search

+19 -8
View File
@@ -26,6 +26,16 @@ export interface EdgeSegment {
file: FileMeta
}
/**
* A copy of a string that doesn't keep the string it was cut from alive: a
* slice of a chunk's text otherwise holds on to the whole text (tens of MB)
* for as long as the file it belongs to stays open. External memory was
* ~570 MB of a worker's 1.1 GB (LOG.md 2026-10-09).
*/
function own(text: string): string {
return (' ' + text).slice(1)
}
interface Counter {
index: number
length: number
@@ -110,14 +120,14 @@ export class SegmentScanner {
let file = this.files.get(key)
if (file === undefined) {
file = this.describe(subject, nameEnd, counter, from, xref, group)
this.files.set(key, file)
this.files.set(own(key), file)
}
if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null
return {
fileKey: file.fileKey,
number: counter.segment,
totalParts: file.totalParts,
messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId,
messageId: own(messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId),
bytes: Number.parseInt(bytes, 10) || 0,
articleNumber,
postedAt: Date.parse(date),
@@ -129,17 +139,18 @@ export class SegmentScanner {
private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
if (nameStart === -1) return null
const name = subject.slice(nameStart + 1, nameEnd)
const name = own(subject.slice(nameStart + 1, nameEnd))
if (!name || counter.total < 1) return null
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim()
const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim())
const poster = own(from)
return {
fileKey: fileKey(from, base, counter.total),
fileKey: fileKey(poster, base, counter.total),
name,
poster: from,
poster,
subject: base,
totalParts: counter.total,
groups: xrefGroups(xref, group),
release: releaseIdentity({ poster: from, subject: base, name, weak: false }),
release: releaseIdentity({ poster, subject: base, name, weak: false }),
}
}
}
@@ -149,7 +160,7 @@ function xrefGroups(xref: string, group: string): string[] {
const tokens = xref.trim().split(/\s+/)
for (let i = 1; i < tokens.length; i++) {
const colon = tokens[i].lastIndexOf(':')
if (colon > 0) groups.push(tokens[i].slice(0, colon))
if (colon > 0) groups.push(own(tokens[i].slice(0, colon)))
}
return groups.length > 0 ? groups : [group]
}