Daan MeijerandClaude Opus 5.5
8c36744088
- seed_nodes (hashed bearer tokens) and seed_ranges (planned ranges with lease, resume point and counts); migration …140. - SeedOrchestrator: lease (oldest plan first, FOR UPDATE SKIP LOCKED; a lease without a batch for 15 min passes to the next node and resumes at next_id), batch (stores files and fragments, moves next_id), finish, release, plan (from below live indexing / the backfill / earlier plans down to N days back; takes over the group's backfill), status. - Seed API under /api/v1/seed (lease, ranges/:id/batches|done|release, status) behind a node-token middleware; JSON bodies up to 64 MB after gzip. - seed:node runs a node: no database or Redis, outgoing HTTPS only, N connections, gzip batches, retries for network errors and 5xx, drops a range on 409, gives its ranges back on SIGTERM. seed-node/ has a compose file that builds the image from this repository. - seed:token, seed:plan, seed:status for the operator. Header parsing moved to lib/ingest and the range loop to lib/seed/runner, shared by seed:range and seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
66 lines
2.5 KiB
TypeScript
66 lines
2.5 KiB
TypeScript
import type { HttpContext } from '@adonisjs/core/http'
|
|
import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator'
|
|
import type { SeedBatch } from '#lib/seed/wire'
|
|
|
|
/**
|
|
* The seed API (CONCEPT.md D38). Nodes make outgoing HTTPS requests only:
|
|
* lease a range, post batches of what they found, then finish or release
|
|
* the range. Bodies may be gzip-compressed (Content-Encoding: gzip).
|
|
*/
|
|
export default class SeedController {
|
|
/** POST /api/v1/seed/lease → 200 { range } or 204 when there's nothing to do. */
|
|
async lease({ seedNode, response }: HttpContext) {
|
|
const range = await SeedOrchestrator.lease(seedNode.id)
|
|
return range ? { range } : response.noContent()
|
|
}
|
|
|
|
/** POST /api/v1/seed/ranges/:id/batches with a SeedBatch. */
|
|
async batch({ seedNode, params, request, response }: HttpContext) {
|
|
const batch = request.body() as SeedBatch
|
|
if (!Number.isSafeInteger(batch?.next) || !Array.isArray(batch.files) || !Array.isArray(batch.fragments)) {
|
|
return response.badRequest({ error: 'Expected { next, headers, files[], fragments[] }' })
|
|
}
|
|
return this.leased(response, async () => {
|
|
const stored = await SeedOrchestrator.batch(seedNode.id, Number(params.id), {
|
|
...batch,
|
|
headers: Number(batch.headers) || 0,
|
|
obfuscated: Number(batch.obfuscated) || 0,
|
|
deferred: Number(batch.deferred) || 0,
|
|
})
|
|
return { stored }
|
|
})
|
|
}
|
|
|
|
/** POST /api/v1/seed/ranges/:id/done */
|
|
async done({ seedNode, params, response }: HttpContext) {
|
|
return this.leased(response, async () => {
|
|
await SeedOrchestrator.finish(seedNode.id, Number(params.id))
|
|
return { ok: true }
|
|
})
|
|
}
|
|
|
|
/** POST /api/v1/seed/ranges/:id/release { error? } */
|
|
async release({ seedNode, params, request, response }: HttpContext) {
|
|
const error = request.input('error')
|
|
return this.leased(response, async () => {
|
|
await SeedOrchestrator.release(seedNode.id, Number(params.id), typeof error === 'string' ? error : undefined)
|
|
return { ok: true }
|
|
})
|
|
}
|
|
|
|
/** GET /api/v1/seed/status */
|
|
async status() {
|
|
return SeedOrchestrator.status()
|
|
}
|
|
|
|
/** 409 when the range was given to another node meanwhile; the node should drop it. */
|
|
private async leased(response: HttpContext['response'], action: () => Promise<unknown>) {
|
|
try {
|
|
return await action()
|
|
} catch (error) {
|
|
if (error instanceof LeaseLost) return response.conflict({ error: error.message })
|
|
throw error
|
|
}
|
|
}
|
|
}
|