Rebuild: strings that outlive their chunk are copied (worker 1.14 → 0.66 GB)
A slice of a chunk's text keeps the whole text alive (tens of MB) for as long as the file it belongs to stays open: message-ids, file details and cache keys are now copies. edge:bench on boneless: peak RSS 1,137 → ~660 MB, external memory 570 → 40–75 MB, the same speed (~190k headers/s); the same results on 800k real lines. edge:bench prints the breakdown. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
80c920f678
commit
73fc9aabdf
2 files changed
+26
-10
No files matched your search
+19
-8
@@ -26,6 +26,16 @@ export interface EdgeSegment {
|
||||
file: FileMeta
|
||||
}
|
||||
|
||||
/**
|
||||
* A copy of a string that doesn't keep the string it was cut from alive: a
|
||||
* slice of a chunk's text otherwise holds on to the whole text (tens of MB)
|
||||
* for as long as the file it belongs to stays open. External memory was
|
||||
* ~570 MB of a worker's 1.1 GB (LOG.md 2026-10-09).
|
||||
*/
|
||||
function own(text: string): string {
|
||||
return (' ' + text).slice(1)
|
||||
}
|
||||
|
||||
interface Counter {
|
||||
index: number
|
||||
length: number
|
||||
@@ -110,14 +120,14 @@ export class SegmentScanner {
|
||||
let file = this.files.get(key)
|
||||
if (file === undefined) {
|
||||
file = this.describe(subject, nameEnd, counter, from, xref, group)
|
||||
this.files.set(key, file)
|
||||
this.files.set(own(key), file)
|
||||
}
|
||||
if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null
|
||||
return {
|
||||
fileKey: file.fileKey,
|
||||
number: counter.segment,
|
||||
totalParts: file.totalParts,
|
||||
messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId,
|
||||
messageId: own(messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId),
|
||||
bytes: Number.parseInt(bytes, 10) || 0,
|
||||
articleNumber,
|
||||
postedAt: Date.parse(date),
|
||||
@@ -129,17 +139,18 @@ export class SegmentScanner {
|
||||
private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
|
||||
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
|
||||
if (nameStart === -1) return null
|
||||
const name = subject.slice(nameStart + 1, nameEnd)
|
||||
const name = own(subject.slice(nameStart + 1, nameEnd))
|
||||
if (!name || counter.total < 1) return null
|
||||
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim()
|
||||
const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim())
|
||||
const poster = own(from)
|
||||
return {
|
||||
fileKey: fileKey(from, base, counter.total),
|
||||
fileKey: fileKey(poster, base, counter.total),
|
||||
name,
|
||||
poster: from,
|
||||
poster,
|
||||
subject: base,
|
||||
totalParts: counter.total,
|
||||
groups: xrefGroups(xref, group),
|
||||
release: releaseIdentity({ poster: from, subject: base, name, weak: false }),
|
||||
release: releaseIdentity({ poster, subject: base, name, weak: false }),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -149,7 +160,7 @@ function xrefGroups(xref: string, group: string): string[] {
|
||||
const tokens = xref.trim().split(/\s+/)
|
||||
for (let i = 1; i < tokens.length; i++) {
|
||||
const colon = tokens[i].lastIndexOf(':')
|
||||
if (colon > 0) groups.push(tokens[i].slice(0, colon))
|
||||
if (colon > 0) groups.push(own(tokens[i].slice(0, colon)))
|
||||
}
|
||||
return groups.length > 0 ? groups : [group]
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user