Rebuild: the segment counter by a backwards scan, one cache from poster+subject to the file
edge:bench (one core): boneless 193k → 215k headers/s, ath 303k → 343k. Same results on 800k real lines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
638cfa1dcd
commit
db882f0b48
1 file changed
+56
-46
+56
-46
@@ -26,13 +26,37 @@ export interface EdgeSegment {
|
||||
file: FileMeta
|
||||
}
|
||||
|
||||
interface ParsedSubjectBase {
|
||||
name: string
|
||||
totalSegments: number
|
||||
base: string
|
||||
interface Counter {
|
||||
index: number
|
||||
length: number
|
||||
segment: number
|
||||
total: number
|
||||
}
|
||||
|
||||
const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
|
||||
/** The last "(k/N)" after `after` (the filename's closing quote), scanning back from the end; null when there is none. */
|
||||
function findCounter(subject: string, after: number): Counter | null {
|
||||
let open = subject.length
|
||||
while (true) {
|
||||
open = subject.lastIndexOf('(', open - 1)
|
||||
if (open <= after) return null
|
||||
let i = open + 1
|
||||
let segment = 0
|
||||
let digits = 0
|
||||
for (let c = subject.charCodeAt(i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) {
|
||||
segment = segment * 10 + (c - 48)
|
||||
digits++
|
||||
}
|
||||
if (digits === 0 || subject.charCodeAt(i) !== 47) continue
|
||||
let total = 0
|
||||
digits = 0
|
||||
for (let c = subject.charCodeAt(++i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) {
|
||||
total = total * 10 + (c - 48)
|
||||
digits++
|
||||
}
|
||||
if (digits === 0 || subject.charCodeAt(i) !== 41) continue
|
||||
return { index: open, length: i + 1 - open, segment, total }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds segments from overview fields (REBUILD.md, edge worker), the same
|
||||
@@ -43,11 +67,10 @@ const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
|
||||
* (LOG.md 2026-10-09). The caches live until `clear`, once per batch.
|
||||
*/
|
||||
export class SegmentScanner {
|
||||
private subjects = new Map<string, ParsedSubjectBase | null>()
|
||||
private files = new Map<string, FileMeta>()
|
||||
/** By poster and subject without its counter: the file's details, or null for a subject that names no file. */
|
||||
private files = new Map<string, FileMeta | null>()
|
||||
|
||||
clear() {
|
||||
this.subjects.clear()
|
||||
this.files.clear()
|
||||
}
|
||||
|
||||
@@ -81,37 +104,18 @@ export class SegmentScanner {
|
||||
if (!subject || !messageId) return null
|
||||
const nameEnd = subject.lastIndexOf('"')
|
||||
if (nameEnd <= 0) return null
|
||||
// The segment counter: the last "(k/N)" after the filename (a file size may follow it).
|
||||
SEGMENT_COUNTER.lastIndex = nameEnd
|
||||
let counter: RegExpExecArray | null = null
|
||||
for (let m = SEGMENT_COUNTER.exec(subject); m; m = SEGMENT_COUNTER.exec(subject)) counter = m
|
||||
const counter = findCounter(subject, nameEnd)
|
||||
if (!counter) return null
|
||||
const key = subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter[0].length)
|
||||
let parsed = this.subjects.get(key)
|
||||
if (parsed === undefined) {
|
||||
parsed = parseBase(subject, nameEnd, counter)
|
||||
this.subjects.set(key, parsed)
|
||||
}
|
||||
if (!parsed) return null
|
||||
const number = Number.parseInt(counter[1], 10)
|
||||
if (number < 1 || number > parsed.totalSegments) return null
|
||||
const id = `${from}\n${parsed.base}\n${parsed.totalSegments}`
|
||||
let file = this.files.get(id)
|
||||
if (!file) {
|
||||
file = {
|
||||
fileKey: fileKey(from, parsed.base, parsed.totalSegments),
|
||||
name: parsed.name,
|
||||
poster: from,
|
||||
subject: parsed.base,
|
||||
totalParts: parsed.totalSegments,
|
||||
groups: xrefGroups(xref, group),
|
||||
release: releaseIdentity({ poster: from, subject: parsed.base, name: parsed.name, weak: false }),
|
||||
}
|
||||
this.files.set(id, file)
|
||||
const key = from + '\u0001' + subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter.length)
|
||||
let file = this.files.get(key)
|
||||
if (file === undefined) {
|
||||
file = this.describe(subject, nameEnd, counter, from, xref, group)
|
||||
this.files.set(key, file)
|
||||
}
|
||||
if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null
|
||||
return {
|
||||
fileKey: file.fileKey,
|
||||
number,
|
||||
number: counter.segment,
|
||||
totalParts: file.totalParts,
|
||||
messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId,
|
||||
bytes: Number.parseInt(bytes, 10) || 0,
|
||||
@@ -120,18 +124,24 @@ export class SegmentScanner {
|
||||
file,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** As parseSubject: the name, the segment count and the base, from a subject whose counter was found. */
|
||||
function parseBase(subject: string, nameEnd: number, counter: RegExpExecArray): ParsedSubjectBase | null {
|
||||
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
|
||||
if (nameStart === -1) return null
|
||||
const name = subject.slice(nameStart + 1, nameEnd)
|
||||
if (!name) return null
|
||||
const totalSegments = Number.parseInt(counter[2], 10)
|
||||
if (totalSegments < 1) return null
|
||||
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length)).replace(/\s+/g, ' ').trim()
|
||||
return { name, totalSegments, base }
|
||||
/** A file's details from its first article seen (as parseSubject and segmentFromHeader). */
|
||||
private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
|
||||
const nameStart = subject.lastIndexOf('"', nameEnd - 1)
|
||||
if (nameStart === -1) return null
|
||||
const name = subject.slice(nameStart + 1, nameEnd)
|
||||
if (!name || counter.total < 1) return null
|
||||
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim()
|
||||
return {
|
||||
fileKey: fileKey(from, base, counter.total),
|
||||
name,
|
||||
poster: from,
|
||||
subject: base,
|
||||
totalParts: counter.total,
|
||||
groups: xrefGroups(xref, group),
|
||||
release: releaseIdentity({ poster: from, subject: base, name, weak: false }),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function xrefGroups(xref: string, group: string): string[] {
|
||||
|
||||
Reference in new issue
Block a user