From db882f0b4876c868defb606c72e50d01fb9e81e0 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 01:16:04 +0200 Subject: [PATCH] Rebuild: the segment counter by a backwards scan, one cache from poster+subject to the file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit edge:bench (one core): boneless 193k → 215k headers/s, ath 303k → 343k. Same results on 800k real lines. Co-Authored-By: Claude Opus 5.5 --- lib/edge/scan.ts | 102 ++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 46 deletions(-) diff --git a/lib/edge/scan.ts b/lib/edge/scan.ts index 864acf4..cee5861 100644 --- a/lib/edge/scan.ts +++ b/lib/edge/scan.ts @@ -26,13 +26,37 @@ export interface EdgeSegment { file: FileMeta } -interface ParsedSubjectBase { - name: string - totalSegments: number - base: string +interface Counter { + index: number + length: number + segment: number + total: number } -const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g +/** The last "(k/N)" after `after` (the filename's closing quote), scanning back from the end; null when there is none. */ +function findCounter(subject: string, after: number): Counter | null { + let open = subject.length + while (true) { + open = subject.lastIndexOf('(', open - 1) + if (open <= after) return null + let i = open + 1 + let segment = 0 + let digits = 0 + for (let c = subject.charCodeAt(i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) { + segment = segment * 10 + (c - 48) + digits++ + } + if (digits === 0 || subject.charCodeAt(i) !== 47) continue + let total = 0 + digits = 0 + for (let c = subject.charCodeAt(++i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) { + total = total * 10 + (c - 48) + digits++ + } + if (digits === 0 || subject.charCodeAt(i) !== 41) continue + return { index: open, length: i + 1 - open, segment, total } + } +} /** * Builds segments from overview fields (REBUILD.md, edge worker), the same @@ -43,11 +67,10 @@ const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g * (LOG.md 2026-10-09). The caches live until `clear`, once per batch. */ export class SegmentScanner { - private subjects = new Map() - private files = new Map() + /** By poster and subject without its counter: the file's details, or null for a subject that names no file. */ + private files = new Map() clear() { - this.subjects.clear() this.files.clear() } @@ -81,37 +104,18 @@ export class SegmentScanner { if (!subject || !messageId) return null const nameEnd = subject.lastIndexOf('"') if (nameEnd <= 0) return null - // The segment counter: the last "(k/N)" after the filename (a file size may follow it). - SEGMENT_COUNTER.lastIndex = nameEnd - let counter: RegExpExecArray | null = null - for (let m = SEGMENT_COUNTER.exec(subject); m; m = SEGMENT_COUNTER.exec(subject)) counter = m + const counter = findCounter(subject, nameEnd) if (!counter) return null - const key = subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter[0].length) - let parsed = this.subjects.get(key) - if (parsed === undefined) { - parsed = parseBase(subject, nameEnd, counter) - this.subjects.set(key, parsed) - } - if (!parsed) return null - const number = Number.parseInt(counter[1], 10) - if (number < 1 || number > parsed.totalSegments) return null - const id = `${from}\n${parsed.base}\n${parsed.totalSegments}` - let file = this.files.get(id) - if (!file) { - file = { - fileKey: fileKey(from, parsed.base, parsed.totalSegments), - name: parsed.name, - poster: from, - subject: parsed.base, - totalParts: parsed.totalSegments, - groups: xrefGroups(xref, group), - release: releaseIdentity({ poster: from, subject: parsed.base, name: parsed.name, weak: false }), - } - this.files.set(id, file) + const key = from + '\u0001' + subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter.length) + let file = this.files.get(key) + if (file === undefined) { + file = this.describe(subject, nameEnd, counter, from, xref, group) + this.files.set(key, file) } + if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null return { fileKey: file.fileKey, - number, + number: counter.segment, totalParts: file.totalParts, messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId, bytes: Number.parseInt(bytes, 10) || 0, @@ -120,18 +124,24 @@ export class SegmentScanner { file, } } -} -/** As parseSubject: the name, the segment count and the base, from a subject whose counter was found. */ -function parseBase(subject: string, nameEnd: number, counter: RegExpExecArray): ParsedSubjectBase | null { - const nameStart = subject.lastIndexOf('"', nameEnd - 1) - if (nameStart === -1) return null - const name = subject.slice(nameStart + 1, nameEnd) - if (!name) return null - const totalSegments = Number.parseInt(counter[2], 10) - if (totalSegments < 1) return null - const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length)).replace(/\s+/g, ' ').trim() - return { name, totalSegments, base } + /** A file's details from its first article seen (as parseSubject and segmentFromHeader). */ + private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null { + const nameStart = subject.lastIndexOf('"', nameEnd - 1) + if (nameStart === -1) return null + const name = subject.slice(nameStart + 1, nameEnd) + if (!name || counter.total < 1) return null + const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim() + return { + fileKey: fileKey(from, base, counter.total), + name, + poster: from, + subject: base, + totalParts: counter.total, + groups: xrefGroups(xref, group), + release: releaseIdentity({ poster: from, subject: base, name, weak: false }), + } + } } function xrefGroups(xref: string, group: string): string[] {