Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker
- groups:add / ranges:plan / nodes:create / coordinator:run commands - the node API: lease, the range's chunks as one framed stream from the spool, batches, heartbeat, finish, release (node guard, token per node) - edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for the rest), posts a batch every 10 chunks, gives its range back on SIGTERM - releases are gathered per batch: 1.04 chunk rows per release at 10 chunks per batch (1.35 when built per XOVER chunk) - partitions are checked per batch instead of remembered Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
20910647fa
commit
d5f168d8d2
36 files changed
+1193
-14
No files matched your search
@@ -0,0 +1,61 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import Spool from '#services/spool'
|
||||
import type { Lease } from '#lib/edge/index'
|
||||
|
||||
export class LeaseLost extends Error {}
|
||||
|
||||
/**
|
||||
* Ranges for workers (REBUILD.md, coordinator): the next range whose first
|
||||
* chunk is spooled, by priority (oldest era first); a lease whose worker
|
||||
* went quiet for LEASE_TTL_SECONDS is taken over and resumes at its resume
|
||||
* point.
|
||||
*/
|
||||
class Leases {
|
||||
async lease(nodeId: number): Promise<Lease | null> {
|
||||
const rules = await RuleSets.current()
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
|
||||
WHERE r.id = (
|
||||
SELECT id FROM ranges
|
||||
WHERE spooled_chunks > 0 AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
|
||||
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
|
||||
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds]
|
||||
)
|
||||
const row = rows[0]
|
||||
if (!row) return null
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
const group = await db.from('groups').where('id', row.group_id).select('name').firstOrFail()
|
||||
return { range: { id: Number(row.id), group: group.name, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) }, rules }
|
||||
}
|
||||
|
||||
async leased(rangeId: number, nodeId: number) {
|
||||
const range = await db.from('ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
return range
|
||||
}
|
||||
|
||||
async heartbeat(rangeId: number, nodeId: number) {
|
||||
await this.leased(rangeId, nodeId)
|
||||
await db.from('ranges').where('id', rangeId).update({ heartbeat_at: new Date() })
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
}
|
||||
|
||||
/** The worker reached the range's end: done, and its spooled chunks go. */
|
||||
async finish(rangeId: number, nodeId: number) {
|
||||
const range = await this.leased(rangeId, nodeId)
|
||||
if (Number(range.next_id) <= Number(range.end_id)) throw new LeaseLost(`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`)
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
|
||||
await Spool.remove(rangeId)
|
||||
}
|
||||
|
||||
/** A stopping worker gives its range back; the next one resumes it. */
|
||||
async release(rangeId: number, nodeId: number) {
|
||||
await this.leased(rangeId, nodeId)
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'planned', node_id: null })
|
||||
}
|
||||
}
|
||||
|
||||
export default new Leases()
|
||||
Reference in new issue
Block a user