Rebuild: the segment counter by a backwards scan, one cache from poster+subject to the file

edge:bench (one core): boneless 193k → 215k headers/s, ath 303k → 343k.
Same results on 800k real lines.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 01:16:04 +02:00
1 parent 638cfa1dcd
commit db882f0b48
1 file changed
+56 -46
+56 -46
View File
@@ -26,13 +26,37 @@ export interface EdgeSegment {
file: FileMeta
}
interface ParsedSubjectBase {
name: string
totalSegments: number
base: string
interface Counter {
index: number
length: number
segment: number
total: number
}
const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
/** The last "(k/N)" after `after` (the filename's closing quote), scanning back from the end; null when there is none. */
function findCounter(subject: string, after: number): Counter | null {
let open = subject.length
while (true) {
open = subject.lastIndexOf('(', open - 1)
if (open <= after) return null
let i = open + 1
let segment = 0
let digits = 0
for (let c = subject.charCodeAt(i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) {
segment = segment * 10 + (c - 48)
digits++
}
if (digits === 0 || subject.charCodeAt(i) !== 47) continue
let total = 0
digits = 0
for (let c = subject.charCodeAt(++i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) {
total = total * 10 + (c - 48)
digits++
}
if (digits === 0 || subject.charCodeAt(i) !== 41) continue
return { index: open, length: i + 1 - open, segment, total }
}
}
/**
* Builds segments from overview fields (REBUILD.md, edge worker), the same
@@ -43,11 +67,10 @@ const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
* (LOG.md 2026-10-09). The caches live until `clear`, once per batch.
*/
export class SegmentScanner {
private subjects = new Map<string, ParsedSubjectBase | null>()
private files = new Map<string, FileMeta>()
/** By poster and subject without its counter: the file's details, or null for a subject that names no file. */
private files = new Map<string, FileMeta | null>()
clear() {
this.subjects.clear()
this.files.clear()
}
@@ -81,37 +104,18 @@ export class SegmentScanner {
if (!subject || !messageId) return null
const nameEnd = subject.lastIndexOf('"')
if (nameEnd <= 0) return null
// The segment counter: the last "(k/N)" after the filename (a file size may follow it).
SEGMENT_COUNTER.lastIndex = nameEnd
let counter: RegExpExecArray | null = null
for (let m = SEGMENT_COUNTER.exec(subject); m; m = SEGMENT_COUNTER.exec(subject)) counter = m
const counter = findCounter(subject, nameEnd)
if (!counter) return null
const key = subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter[0].length)
let parsed = this.subjects.get(key)
if (parsed === undefined) {
parsed = parseBase(subject, nameEnd, counter)
this.subjects.set(key, parsed)
}
if (!parsed) return null
const number = Number.parseInt(counter[1], 10)
if (number < 1 || number > parsed.totalSegments) return null
const id = `${from}\n${parsed.base}\n${parsed.totalSegments}`
let file = this.files.get(id)
if (!file) {
file = {
fileKey: fileKey(from, parsed.base, parsed.totalSegments),
name: parsed.name,
poster: from,
subject: parsed.base,
totalParts: parsed.totalSegments,
groups: xrefGroups(xref, group),
release: releaseIdentity({ poster: from, subject: parsed.base, name: parsed.name, weak: false }),
}
this.files.set(id, file)
const key = from + '\u0001' + subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter.length)
let file = this.files.get(key)
if (file === undefined) {
file = this.describe(subject, nameEnd, counter, from, xref, group)
this.files.set(key, file)
}
if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null
return {
fileKey: file.fileKey,
number,
number: counter.segment,
totalParts: file.totalParts,
messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId,
bytes: Number.parseInt(bytes, 10) || 0,
@@ -120,18 +124,24 @@ export class SegmentScanner {
file,
}
}
}
/** As parseSubject: the name, the segment count and the base, from a subject whose counter was found. */
function parseBase(subject: string, nameEnd: number, counter: RegExpExecArray): ParsedSubjectBase | null {
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
if (nameStart === -1) return null
const name = subject.slice(nameStart + 1, nameEnd)
if (!name) return null
const totalSegments = Number.parseInt(counter[2], 10)
if (totalSegments < 1) return null
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length)).replace(/\s+/g, ' ').trim()
return { name, totalSegments, base }
/** A file's details from its first article seen (as parseSubject and segmentFromHeader). */
private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
if (nameStart === -1) return null
const name = subject.slice(nameStart + 1, nameEnd)
if (!name || counter.total < 1) return null
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim()
return {
fileKey: fileKey(from, base, counter.total),
name,
poster: from,
subject: base,
totalParts: counter.total,
groups: xrefGroups(xref, group),
release: releaseIdentity({ poster: from, subject: base, name, weak: false }),
}
}
}
function xrefGroups(xref: string, group: string): string[] {