Daan MeijerandClaude Opus 5.5
d5f168d8d2
- groups:add / ranges:plan / nodes:create / coordinator:run commands - the node API: lease, the range's chunks as one framed stream from the spool, batches, heartbeat, finish, release (node guard, token per node) - edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for the rest), posts a batch every 10 chunks, gives its range back on SIGTERM - releases are gathered per batch: 1.04 chunk rows per release at 10 chunks per batch (1.35 when built per XOVER chunk) - partitions are checked per batch instead of remembered Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
36 lines
1.3 KiB
TypeScript
36 lines
1.3 KiB
TypeScript
/**
|
|
* A range's chunks on the wire (REBUILD.md, prefetch): one streaming
|
|
* response, a frame per chunk: from (8 bytes), to (8 bytes), length
|
|
* (4 bytes), then the chunk as the news server sent it (compressed XOVER).
|
|
*/
|
|
export interface Frame {
|
|
from: number
|
|
to: number
|
|
data: Buffer
|
|
}
|
|
|
|
const HEADER = 20
|
|
|
|
export function encodeFrame(frame: Frame): Buffer {
|
|
const head = Buffer.alloc(HEADER)
|
|
head.writeBigUInt64BE(BigInt(frame.from), 0)
|
|
head.writeBigUInt64BE(BigInt(frame.to), 8)
|
|
head.writeUInt32BE(frame.data.length, 16)
|
|
return Buffer.concat([head, frame.data])
|
|
}
|
|
|
|
/** The frames of a byte stream, as they complete. */
|
|
export async function* decodeFrames(stream: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): AsyncGenerator<Frame> {
|
|
let buffer = Buffer.alloc(0)
|
|
for await (const piece of stream) {
|
|
buffer = buffer.length === 0 ? Buffer.from(piece) : Buffer.concat([buffer, piece])
|
|
while (buffer.length >= HEADER) {
|
|
const length = buffer.readUInt32BE(16)
|
|
if (buffer.length < HEADER + length) break
|
|
yield { from: Number(buffer.readBigUInt64BE(0)), to: Number(buffer.readBigUInt64BE(8)), data: buffer.subarray(HEADER, HEADER + length) }
|
|
buffer = buffer.subarray(HEADER + length)
|
|
}
|
|
}
|
|
if (buffer.length > 0) throw new Error(`Stream ended inside a frame (${buffer.length} bytes left)`)
|
|
}
|