Daan MeijerandClaude Opus 5.5
fa7d4f3394
End to end (workstation, recent a.b.boneless ranges from the spool): a 2M range in 10–12 s, 166k–202k headers/s per worker process (edge:bench on one core: 215k). - Prefetcher: spooled_at was set against a total counted from the range's resume point, which shrinks as the worker advances: the last chunks were never fetched and the worker waited. Now counted against the whole range, only new chunks counted, and leased ranges are always checked against the files. Planned ranges ahead are bounded (it had spooled ever more); the spool size is measured every 30 s (a stat per file). - The stream sends an empty frame every 20 s while waiting for a chunk (Node's fetch gave up on a quiet body after 5 minutes). - A range taken over midway: the first batch starts at the resume point (it started at the range's start and was refused). - edge:node inflates the next chunk on the thread pool while indexing the current one, and doesn't wait for a post before indexing on (posts stay in order; a failed post stops the range). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
43 lines
1.6 KiB
TypeScript
43 lines
1.6 KiB
TypeScript
/**
|
|
* A range's chunks on the wire (REBUILD.md, prefetch): one streaming
|
|
* response, a frame per chunk: from (8 bytes), to (8 bytes), length
|
|
* (4 bytes), then the chunk as the news server sent it (compressed XOVER).
|
|
*/
|
|
export interface Frame {
|
|
from: number
|
|
to: number
|
|
data: Buffer
|
|
}
|
|
|
|
const HEADER = 20
|
|
|
|
/** An empty frame, sent while the next chunk is still being fetched, so the response doesn't go quiet. */
|
|
export const KEEPALIVE: Frame = { from: 0, to: 0, data: Buffer.alloc(0) }
|
|
|
|
export function isKeepalive(frame: Frame) {
|
|
return frame.data.length === 0 && frame.from === 0
|
|
}
|
|
|
|
export function encodeFrame(frame: Frame): Buffer {
|
|
const head = Buffer.alloc(HEADER)
|
|
head.writeBigUInt64BE(BigInt(frame.from), 0)
|
|
head.writeBigUInt64BE(BigInt(frame.to), 8)
|
|
head.writeUInt32BE(frame.data.length, 16)
|
|
return Buffer.concat([head, frame.data])
|
|
}
|
|
|
|
/** The frames of a byte stream, as they complete. */
|
|
export async function* decodeFrames(stream: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): AsyncGenerator<Frame> {
|
|
let buffer = Buffer.alloc(0)
|
|
for await (const piece of stream) {
|
|
buffer = buffer.length === 0 ? Buffer.from(piece) : Buffer.concat([buffer, piece])
|
|
while (buffer.length >= HEADER) {
|
|
const length = buffer.readUInt32BE(16)
|
|
if (buffer.length < HEADER + length) break
|
|
yield { from: Number(buffer.readBigUInt64BE(0)), to: Number(buffer.readBigUInt64BE(8)), data: buffer.subarray(HEADER, HEADER + length) }
|
|
buffer = buffer.subarray(HEADER + length)
|
|
}
|
|
}
|
|
if (buffer.length > 0) throw new Error(`Stream ended inside a frame (${buffer.length} bytes left)`)
|
|
}
|