Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker
- groups:add / ranges:plan / nodes:create / coordinator:run commands - the node API: lease, the range's chunks as one framed stream from the spool, batches, heartbeat, finish, release (node guard, token per node) - edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for the rest), posts a batch every 10 chunks, gives its range back on SIGTERM - releases are gathered per batch: 1.04 chunk rows per release at 10 chunks per batch (1.35 when built per XOVER chunk) - partitions are checked per batch instead of remembered Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
20910647fa
commit
d5f168d8d2
36 files changed
+1193
-14
No files matched your search
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* A range's chunks on the wire (REBUILD.md, prefetch): one streaming
|
||||
* response, a frame per chunk: from (8 bytes), to (8 bytes), length
|
||||
* (4 bytes), then the chunk as the news server sent it (compressed XOVER).
|
||||
*/
|
||||
export interface Frame {
|
||||
from: number
|
||||
to: number
|
||||
data: Buffer
|
||||
}
|
||||
|
||||
const HEADER = 20
|
||||
|
||||
export function encodeFrame(frame: Frame): Buffer {
|
||||
const head = Buffer.alloc(HEADER)
|
||||
head.writeBigUInt64BE(BigInt(frame.from), 0)
|
||||
head.writeBigUInt64BE(BigInt(frame.to), 8)
|
||||
head.writeUInt32BE(frame.data.length, 16)
|
||||
return Buffer.concat([head, frame.data])
|
||||
}
|
||||
|
||||
/** The frames of a byte stream, as they complete. */
|
||||
export async function* decodeFrames(stream: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): AsyncGenerator<Frame> {
|
||||
let buffer = Buffer.alloc(0)
|
||||
for await (const piece of stream) {
|
||||
buffer = buffer.length === 0 ? Buffer.from(piece) : Buffer.concat([buffer, piece])
|
||||
while (buffer.length >= HEADER) {
|
||||
const length = buffer.readUInt32BE(16)
|
||||
if (buffer.length < HEADER + length) break
|
||||
yield { from: Number(buffer.readBigUInt64BE(0)), to: Number(buffer.readBigUInt64BE(8)), data: buffer.subarray(HEADER, HEADER + length) }
|
||||
buffer = buffer.subarray(HEADER + length)
|
||||
}
|
||||
}
|
||||
if (buffer.length > 0) throw new Error(`Stream ended inside a frame (${buffer.length} bytes left)`)
|
||||
}
|
||||
Reference in new issue
Block a user