Daan MeijerandClaude Opus 5.5
8c36744088
- seed_nodes (hashed bearer tokens) and seed_ranges (planned ranges with lease, resume point and counts); migration …140. - SeedOrchestrator: lease (oldest plan first, FOR UPDATE SKIP LOCKED; a lease without a batch for 15 min passes to the next node and resumes at next_id), batch (stores files and fragments, moves next_id), finish, release, plan (from below live indexing / the backfill / earlier plans down to N days back; takes over the group's backfill), status. - Seed API under /api/v1/seed (lease, ranges/:id/batches|done|release, status) behind a node-token middleware; JSON bodies up to 64 MB after gzip. - seed:node runs a node: no database or Redis, outgoing HTTPS only, N connections, gzip batches, retries for network errors and 5xx, drops a range on 409, gives its ranges back on SIGTERM. seed-node/ has a compose file that builds the image from this repository. - seed:token, seed:plan, seed:status for the operator. Header parsing moved to lib/ingest and the range loop to lib/seed/runner, shared by seed:range and seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
2.7 KiB
TypeScript
74 lines
2.7 KiB
TypeScript
import type { IngestSegment } from '../ingest/segment.js'
|
|
import type { ReleaseIdentity } from '../release/index.js'
|
|
import type { AggregatedFile } from '../aggregate/index.js'
|
|
|
|
/**
|
|
* A complete file as a seeding node reports it (CONCEPT.md D37): what the
|
|
* file, its release and its binary need, once per file, and the segments
|
|
* as [part number, message-id, bytes].
|
|
*/
|
|
export interface SeedFile {
|
|
fileKey: string
|
|
name: string
|
|
poster: string
|
|
subject: string
|
|
totalParts: number
|
|
groups: string[]
|
|
/** Earliest post date of its articles, ISO 8601. */
|
|
postedAt: string | null
|
|
release: ReleaseIdentity | null
|
|
segments: [number, string, number][]
|
|
}
|
|
|
|
/** A segment of a file that didn't complete on the node, for the regular ingest path. */
|
|
export type SeedFragment = Omit<IngestSegment, 'postedAt' | 'articleNumber'> & { postedAt?: string; articleNumber?: number }
|
|
|
|
/** What a node sends per batch: the article it has reached, and what it found up to there. */
|
|
export interface SeedBatch {
|
|
/** First article number not covered yet (everything below it, from the range start, is in this or earlier batches). */
|
|
next: number
|
|
headers: number
|
|
obfuscated: number
|
|
deferred: number
|
|
files: SeedFile[]
|
|
fragments: SeedFragment[]
|
|
}
|
|
|
|
export function toSeedFile(file: AggregatedFile<IngestSegment>): SeedFile {
|
|
const first = file.segments[0]
|
|
const times = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t))
|
|
return {
|
|
fileKey: file.fileKey,
|
|
name: first.name,
|
|
poster: first.poster,
|
|
subject: first.subject,
|
|
totalParts: first.totalParts,
|
|
groups: first.groups,
|
|
postedAt: times.length > 0 ? new Date(Math.min(...times)).toISOString() : null,
|
|
release: first.release ?? null,
|
|
segments: file.segments.map((s) => [s.number, s.messageId, s.bytes]),
|
|
}
|
|
}
|
|
|
|
export function toFragment(segment: IngestSegment): SeedFragment {
|
|
const { postedAt, articleNumber, ...rest } = segment
|
|
const date = postedAt && !Number.isNaN(postedAt.getTime()) ? postedAt.toISOString() : undefined
|
|
return { ...rest, postedAt: date, articleNumber: articleNumber === undefined ? undefined : Number(articleNumber) }
|
|
}
|
|
|
|
export function fromFragment(fragment: SeedFragment): IngestSegment {
|
|
// The JSON body parser turns empty strings into null; the columns want ''.
|
|
return {
|
|
...fragment,
|
|
poster: fragment.poster ?? '',
|
|
subject: fragment.subject ?? '',
|
|
name: fragment.name ?? '',
|
|
postedAt: fragment.postedAt ? new Date(fragment.postedAt) : undefined,
|
|
}
|
|
}
|
|
|
|
/** A file from a node's JSON, with empty strings back where the body parser made them null. */
|
|
export function normalizeSeedFile(file: SeedFile): SeedFile {
|
|
return { ...file, name: file.name ?? '', poster: file.poster ?? '', subject: file.subject ?? '' }
|
|
}
|