Seed orchestrator: remote nodes lease ranges and post results over HTTPS
- seed_nodes (hashed bearer tokens) and seed_ranges (planned ranges with lease, resume point and counts); migration …140. - SeedOrchestrator: lease (oldest plan first, FOR UPDATE SKIP LOCKED; a lease without a batch for 15 min passes to the next node and resumes at next_id), batch (stores files and fragments, moves next_id), finish, release, plan (from below live indexing / the backfill / earlier plans down to N days back; takes over the group's backfill), status. - Seed API under /api/v1/seed (lease, ranges/:id/batches|done|release, status) behind a node-token middleware; JSON bodies up to 64 MB after gzip. - seed:node runs a node: no database or Redis, outgoing HTTPS only, N connections, gzip batches, retries for network errors and 5xx, drops a range on 409, gives its ranges back on SIGTERM. seed-node/ has a compose file that builds the image from this repository. - seed:token, seed:plan, seed:status for the operator. Header parsing moved to lib/ingest and the range loop to lib/seed/runner, shared by seed:range and seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
97bdf8e982
commit
8c36744088
24 files changed
+1198
-246
No files matched your search
@@ -0,0 +1,73 @@
|
||||
import type { IngestSegment } from '../ingest/segment.js'
|
||||
import type { ReleaseIdentity } from '../release/index.js'
|
||||
import type { AggregatedFile } from '../aggregate/index.js'
|
||||
|
||||
/**
|
||||
* A complete file as a seeding node reports it (CONCEPT.md D37): what the
|
||||
* file, its release and its binary need, once per file, and the segments
|
||||
* as [part number, message-id, bytes].
|
||||
*/
|
||||
export interface SeedFile {
|
||||
fileKey: string
|
||||
name: string
|
||||
poster: string
|
||||
subject: string
|
||||
totalParts: number
|
||||
groups: string[]
|
||||
/** Earliest post date of its articles, ISO 8601. */
|
||||
postedAt: string | null
|
||||
release: ReleaseIdentity | null
|
||||
segments: [number, string, number][]
|
||||
}
|
||||
|
||||
/** A segment of a file that didn't complete on the node, for the regular ingest path. */
|
||||
export type SeedFragment = Omit<IngestSegment, 'postedAt' | 'articleNumber'> & { postedAt?: string; articleNumber?: number }
|
||||
|
||||
/** What a node sends per batch: the article it has reached, and what it found up to there. */
|
||||
export interface SeedBatch {
|
||||
/** First article number not covered yet (everything below it, from the range start, is in this or earlier batches). */
|
||||
next: number
|
||||
headers: number
|
||||
obfuscated: number
|
||||
deferred: number
|
||||
files: SeedFile[]
|
||||
fragments: SeedFragment[]
|
||||
}
|
||||
|
||||
export function toSeedFile(file: AggregatedFile<IngestSegment>): SeedFile {
|
||||
const first = file.segments[0]
|
||||
const times = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t))
|
||||
return {
|
||||
fileKey: file.fileKey,
|
||||
name: first.name,
|
||||
poster: first.poster,
|
||||
subject: first.subject,
|
||||
totalParts: first.totalParts,
|
||||
groups: first.groups,
|
||||
postedAt: times.length > 0 ? new Date(Math.min(...times)).toISOString() : null,
|
||||
release: first.release ?? null,
|
||||
segments: file.segments.map((s) => [s.number, s.messageId, s.bytes]),
|
||||
}
|
||||
}
|
||||
|
||||
export function toFragment(segment: IngestSegment): SeedFragment {
|
||||
const { postedAt, articleNumber, ...rest } = segment
|
||||
const date = postedAt && !Number.isNaN(postedAt.getTime()) ? postedAt.toISOString() : undefined
|
||||
return { ...rest, postedAt: date, articleNumber: articleNumber === undefined ? undefined : Number(articleNumber) }
|
||||
}
|
||||
|
||||
export function fromFragment(fragment: SeedFragment): IngestSegment {
|
||||
// The JSON body parser turns empty strings into null; the columns want ''.
|
||||
return {
|
||||
...fragment,
|
||||
poster: fragment.poster ?? '',
|
||||
subject: fragment.subject ?? '',
|
||||
name: fragment.name ?? '',
|
||||
postedAt: fragment.postedAt ? new Date(fragment.postedAt) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
/** A file from a node's JSON, with empty strings back where the body parser made them null. */
|
||||
export function normalizeSeedFile(file: SeedFile): SeedFile {
|
||||
return { ...file, name: file.name ?? '', poster: file.poster ?? '', subject: file.subject ?? '' }
|
||||
}
|
||||
Reference in new issue
Block a user