Daan MeijerandClaude Opus 5.5
8c36744088
- seed_nodes (hashed bearer tokens) and seed_ranges (planned ranges with lease, resume point and counts); migration …140. - SeedOrchestrator: lease (oldest plan first, FOR UPDATE SKIP LOCKED; a lease without a batch for 15 min passes to the next node and resumes at next_id), batch (stores files and fragments, moves next_id), finish, release, plan (from below live indexing / the backfill / earlier plans down to N days back; takes over the group's backfill), status. - Seed API under /api/v1/seed (lease, ranges/:id/batches|done|release, status) behind a node-token middleware; JSON bodies up to 64 MB after gzip. - seed:node runs a node: no database or Redis, outgoing HTTPS only, N connections, gzip batches, retries for network errors and 5xx, drops a range on 409, gives its ranges back on SIGTERM. seed-node/ has a compose file that builds the image from this repository. - seed:token, seed:plan, seed:status for the operator. Header parsing moved to lib/ingest and the range loop to lib/seed/runner, shared by seed:range and seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
48 lines
2.1 KiB
TypeScript
48 lines
2.1 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
|
|
/**
|
|
* The seed orchestrator (CONCEPT.md D38): nodes that lease article ranges
|
|
* over HTTPS and submit what they aggregated (D37), and the ranges planned
|
|
* for them. A range's id is its priority: plans insert newest first.
|
|
*/
|
|
export default class extends BaseSchema {
|
|
async up() {
|
|
this.schema.createTable('seed_nodes', (table) => {
|
|
table.increments('id')
|
|
table.text('name').notNullable().unique()
|
|
// sha256 of the bearer token; the token itself is shown once, when it's made.
|
|
table.string('token_hash', 64).notNullable().unique()
|
|
table.boolean('disabled').notNullable().defaultTo(false)
|
|
table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
|
|
table.timestamp('last_seen_at', { useTz: true }).nullable()
|
|
})
|
|
|
|
this.schema.createTable('seed_ranges', (table) => {
|
|
table.bigIncrements('id')
|
|
table.text('group_name').notNullable()
|
|
table.bigInteger('start_id').notNullable()
|
|
table.bigInteger('end_id').notNullable()
|
|
// First article not reported yet; a re-leased range resumes here.
|
|
table.bigInteger('next_id').notNullable()
|
|
table.string('status', 8).notNullable().defaultTo('queued')
|
|
table.integer('node_id').nullable().references('id').inTable('seed_nodes').onDelete('SET NULL')
|
|
table.integer('attempts').notNullable().defaultTo(0)
|
|
table.timestamp('leased_at', { useTz: true }).nullable()
|
|
table.timestamp('heartbeat_at', { useTz: true }).nullable()
|
|
table.timestamp('done_at', { useTz: true }).nullable()
|
|
table.bigInteger('headers').notNullable().defaultTo(0)
|
|
table.bigInteger('files').notNullable().defaultTo(0)
|
|
table.bigInteger('fragments').notNullable().defaultTo(0)
|
|
table.text('error').nullable()
|
|
table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
|
|
table.index(['group_name', 'start_id'])
|
|
})
|
|
this.schema.raw(`CREATE INDEX seed_ranges_open_idx ON seed_ranges (id) WHERE status IN ('queued', 'leased')`)
|
|
}
|
|
|
|
async down() {
|
|
this.schema.dropTable('seed_ranges')
|
|
this.schema.dropTable('seed_nodes')
|
|
}
|
|
}
|