diff --git a/commands/edge_bench.ts b/commands/edge_bench.ts index 10e38ef..423ce0f 100644 --- a/commands/edge_bench.ts +++ b/commands/edge_bench.ts @@ -97,6 +97,7 @@ export default class EdgeBench extends BaseCommand { let compressed = 0 let text = 0 let peakRss = 0 + let peakDetail = '' const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 } const counts = new Map() const keys = new Set() @@ -156,14 +157,18 @@ export default class EdgeBench extends BaseCommand { out.fragments += batch.fragments.length out.samples += batch.samples.length for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles) - peakRss = Math.max(peakRss, process.memoryUsage().rss) + const m = process.memoryUsage() + if (m.rss > peakRss) { + peakRss = m.rss + peakDetail = `heap ${(m.heapUsed / 1e6).toFixed(0)}/${(m.heapTotal / 1e6).toFixed(0)} MB, external ${(m.external / 1e6).toFixed(0)} MB, array buffers ${(m.arrayBuffers / 1e6).toFixed(0)} MB` + } } const cpu = process.cpuUsage(cpuStart) const cpuS = (cpu.user + cpu.system) / 1e6 const total = t.inflate + t.parse + t.index this.logger.info(`${group} ${start}–${end}: ${files.length} chunks, ${(compressed / 1e6).toFixed(0)} MB compressed, ${(text / 1e6).toFixed(0)} MB text`) this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`) - this.logger.info(` inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB`) + this.logger.info(` inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`) this.logger.info(` ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`) this.logger.info(` kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`) for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(` dropped ${k}: ${v.toLocaleString('en')} articles`) diff --git a/lib/edge/scan.ts b/lib/edge/scan.ts index cee5861..132c5ca 100644 --- a/lib/edge/scan.ts +++ b/lib/edge/scan.ts @@ -26,6 +26,16 @@ export interface EdgeSegment { file: FileMeta } +/** + * A copy of a string that doesn't keep the string it was cut from alive: a + * slice of a chunk's text otherwise holds on to the whole text (tens of MB) + * for as long as the file it belongs to stays open. External memory was + * ~570 MB of a worker's 1.1 GB (LOG.md 2026-10-09). + */ +function own(text: string): string { + return (' ' + text).slice(1) +} + interface Counter { index: number length: number @@ -110,14 +120,14 @@ export class SegmentScanner { let file = this.files.get(key) if (file === undefined) { file = this.describe(subject, nameEnd, counter, from, xref, group) - this.files.set(key, file) + this.files.set(own(key), file) } if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null return { fileKey: file.fileKey, number: counter.segment, totalParts: file.totalParts, - messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId, + messageId: own(messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId), bytes: Number.parseInt(bytes, 10) || 0, articleNumber, postedAt: Date.parse(date), @@ -129,17 +139,18 @@ export class SegmentScanner { private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null { const nameStart = subject.lastIndexOf('"', nameEnd - 1) if (nameStart === -1) return null - const name = subject.slice(nameStart + 1, nameEnd) + const name = own(subject.slice(nameStart + 1, nameEnd)) if (!name || counter.total < 1) return null - const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim() + const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim()) + const poster = own(from) return { - fileKey: fileKey(from, base, counter.total), + fileKey: fileKey(poster, base, counter.total), name, - poster: from, + poster, subject: base, totalParts: counter.total, groups: xrefGroups(xref, group), - release: releaseIdentity({ poster: from, subject: base, name, weak: false }), + release: releaseIdentity({ poster, subject: base, name, weak: false }), } } } @@ -149,7 +160,7 @@ function xrefGroups(xref: string, group: string): string[] { const tokens = xref.trim().split(/\s+/) for (let i = 1; i < tokens.length; i++) { const colon = tokens[i].lastIndexOf(':') - if (colon > 0) groups.push(tokens[i].slice(0, colon)) + if (colon > 0) groups.push(own(tokens[i].slice(0, colon))) } return groups.length > 0 ? groups : [group] }