Daan MeijerandClaude Opus 5.5
288a6bab8d
One process ran everything on one core. SEED_PROCESSES (default: the number of cores) worker processes now each run seed:node with a share of the slots; stop signals are passed on and crashed workers restarted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
439 lines
18 KiB
TypeScript
439 lines
18 KiB
TypeScript
import { gzipSync, unzipSync } from 'node:zlib'
|
||
import { spawn, type ChildProcess } from 'node:child_process'
|
||
import { availableParallelism } from 'node:os'
|
||
import { setTimeout as sleep } from 'node:timers/promises'
|
||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||
import nntpConfig from '#config/nntp'
|
||
import indexerConfig from '#config/indexer'
|
||
import { XoverClient, parseOverviews, type Overview } from '#lib/nntp/xover_client'
|
||
import { runRange, type OverviewSource } from '#lib/seed/runner'
|
||
import type { SeedBatch } from '#lib/seed/wire'
|
||
import type { OverviewHeader } from '#lib/ingest/segment'
|
||
import { probeInspection, probeWalk, type ContentTask } from '#lib/content/probe'
|
||
|
||
class LeaseLost extends Error {}
|
||
class Stopping extends Error {}
|
||
|
||
interface InspectJob {
|
||
queue: string
|
||
jobId: string
|
||
token: string
|
||
task: ContentTask
|
||
}
|
||
|
||
interface Lease {
|
||
id: number
|
||
group: string
|
||
start: number
|
||
end: number
|
||
}
|
||
|
||
/**
|
||
* A seeding node (CONCEPT.md D38): leases article ranges from the seed API
|
||
* over HTTPS, fetches and aggregates their headers over its own NNTP
|
||
* connection(s) (D37), and posts the results back in gzip-compressed
|
||
* batches. Outgoing requests only: no ports, database or Redis needed.
|
||
*
|
||
* SEED_URL=https://seed.example SEED_TOKEN=sn_… node ace seed:node --ranges 4 --inspectors 2
|
||
*/
|
||
export default class SeedNode extends BaseCommand {
|
||
public static commandName = 'seed:node'
|
||
public static description =
|
||
'Runs a seeding node: leases ranges from the seed API and submits what it aggregated.'
|
||
|
||
public static options: CommandOptions = {
|
||
startApp: false,
|
||
staysAlive: true,
|
||
}
|
||
|
||
@flags.string({ description: 'Seed API base URL (default: SEED_URL)' })
|
||
declare url?: string
|
||
|
||
@flags.string({ description: 'Node token (default: SEED_TOKEN)' })
|
||
declare token?: string
|
||
|
||
@flags.number({
|
||
description:
|
||
'Ranges seeded at once (with SEED_FETCH=direct, one NNTP connection each); 0 for an inspection-only node (default: SEED_RANGES or 1)',
|
||
default: process.env.SEED_RANGES ? Number(process.env.SEED_RANGES) : 1,
|
||
})
|
||
declare ranges: number
|
||
|
||
@flags.number({
|
||
description:
|
||
'Content inspections and archive walks worked on at once, articles through the seed API (default: SEED_INSPECTORS or 0)',
|
||
default: Number(process.env.SEED_INSPECTORS) || 0,
|
||
})
|
||
declare inspectors: number
|
||
|
||
@flags.number({ description: 'Articles per XOVER', default: 20_000 })
|
||
declare chunk: number
|
||
|
||
@flags.number({ description: 'XOVER chunks per batch posted to the API', default: 5 })
|
||
declare submitEvery: number
|
||
|
||
@flags.string({
|
||
description:
|
||
'relay: headers come through the seed API (no NNTP account needed); direct: own NNTP connections (default: SEED_FETCH or relay)',
|
||
default: process.env.SEED_FETCH || 'relay',
|
||
})
|
||
declare fetch: string
|
||
|
||
@flags.number({
|
||
description:
|
||
'Processes the ranges and inspections are spread over; one process uses one core (default: SEED_PROCESSES or the number of cores)',
|
||
default: process.env.SEED_PROCESSES ? Number(process.env.SEED_PROCESSES) : availableParallelism(),
|
||
})
|
||
declare processes: number
|
||
|
||
@flags.boolean({ description: 'Stop when there is nothing to lease instead of waiting' })
|
||
declare once: boolean
|
||
|
||
private stopping = false
|
||
/** In a worker process: its number, in front of its log lines. */
|
||
private readonly child = process.env.SEED_PROCESS ?? ''
|
||
|
||
public async run() {
|
||
const base = (this.url ?? process.env.SEED_URL ?? '').replace(/\/+$/, '')
|
||
const token = this.token ?? process.env.SEED_TOKEN ?? ''
|
||
if (!base || !token) {
|
||
this.logger.error('Set --url and --token (or SEED_URL and SEED_TOKEN).')
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
// JavaScript runs on one core per process: several slots are spread over worker processes.
|
||
const processes = Math.max(1, Math.min(this.processes || 1, this.ranges + this.inspectors))
|
||
if (!this.child && processes > 1) return this.supervise(base, token, processes)
|
||
|
||
const stop = () => {
|
||
if (!this.stopping && !this.child) this.logger.info('Stopping: giving back the ranges in progress…')
|
||
this.stopping = true
|
||
}
|
||
process.on('SIGTERM', stop)
|
||
process.on('SIGINT', stop)
|
||
|
||
const api = new SeedApi(base, token, (message) => this.logger.warning(message))
|
||
if (!this.child) this.logger.info(
|
||
`Seeding node for ${base}: ${this.ranges} range(s) at once, ${this.fetch === 'direct' ? 'own NNTP connections' : 'headers through the seed API'}` +
|
||
(this.inspectors > 0 ? `; ${this.inspectors} inspection(s) at once, articles through the seed API.` : '.')
|
||
)
|
||
await Promise.all([
|
||
...Array.from({ length: this.ranges }, (_, i) => this.work(api, i + 1)),
|
||
...Array.from({ length: this.inspectors }, (_, i) => this.inspect(api, i + 1)),
|
||
])
|
||
if (!this.child) this.logger.info('Seeding node stopped.')
|
||
}
|
||
|
||
/**
|
||
* Runs the slots in `processes` worker processes (this command again, with
|
||
* SEED_PROCESS set), each with its share of the ranges and inspections. A
|
||
* worker that exits while the node runs is restarted; SIGTERM/SIGINT are
|
||
* passed on, and the node stops when all workers have given their work back.
|
||
*/
|
||
private async supervise(base: string, token: string, processes: number) {
|
||
// Slots dealt out in turn (ranges first, then inspections), so every process has work.
|
||
const share = (from: number, to: number, i: number) =>
|
||
Array.from({ length: to - from }, (_, k) => from + k).filter((k) => k % processes === i).length
|
||
this.logger.info(
|
||
`Seeding node for ${base}: ${this.ranges} range(s) and ${this.inspectors} inspection(s) at once, over ${processes} processes, ` +
|
||
`${this.fetch === 'direct' ? 'own NNTP connections' : 'articles and headers through the seed API'}.`
|
||
)
|
||
const children = new Set<ChildProcess>()
|
||
const start = (n: number): Promise<void> =>
|
||
new Promise((resolve) => {
|
||
const args = [
|
||
...process.execArgv,
|
||
process.argv[1],
|
||
'seed:node',
|
||
`--ranges=${share(0, this.ranges, n - 1)}`,
|
||
`--inspectors=${share(this.ranges, this.ranges + this.inspectors, n - 1)}`,
|
||
`--fetch=${this.fetch}`,
|
||
`--chunk=${this.chunk}`,
|
||
`--submit-every=${this.submitEvery}`,
|
||
...(this.once ? ['--once'] : []),
|
||
]
|
||
const child = spawn(process.execPath, args, {
|
||
env: { ...process.env, SEED_URL: base, SEED_TOKEN: token, SEED_PROCESS: String(n) },
|
||
stdio: 'inherit',
|
||
})
|
||
children.add(child)
|
||
child.on('exit', (code, signal) => {
|
||
children.delete(child)
|
||
if (this.stopping || this.once || code === 0) return resolve()
|
||
this.logger.warning(`Process ${n} exited (${signal ?? code}); restarting in 10 s`)
|
||
setTimeout(() => start(n).then(resolve), 10_000)
|
||
})
|
||
})
|
||
const stop = (signal: NodeJS.Signals) => {
|
||
if (!this.stopping) this.logger.info('Stopping: giving back the ranges and inspections in progress…')
|
||
this.stopping = true
|
||
for (const child of children) child.kill(signal)
|
||
}
|
||
process.on('SIGTERM', () => stop('SIGTERM'))
|
||
process.on('SIGINT', () => stop('SIGINT'))
|
||
await Promise.all(Array.from({ length: processes }, (_, i) => start(i + 1)))
|
||
this.logger.info('Seeding node stopped.')
|
||
}
|
||
|
||
private async work(api: SeedApi, worker: number) {
|
||
const log = (message: string) => this.logger.info(`[${this.child && `${this.child}/`}${worker}] ${message}`)
|
||
// The NNTP connection is opened for a lease and kept while leases keep coming; with
|
||
// nothing to lease it's closed, so an idle node holds no connections.
|
||
let client: XoverClient | undefined
|
||
try {
|
||
while (!this.stopping) {
|
||
const lease = await this.lease(api, log)
|
||
if (!lease) {
|
||
client?.close()
|
||
client = undefined
|
||
if (this.once) return
|
||
await this.pause(60_000)
|
||
continue
|
||
}
|
||
if (this.fetch !== 'direct') {
|
||
await this.workRange(api, new RelaySource(api, lease.id), lease, log)
|
||
continue
|
||
}
|
||
if (!client) {
|
||
try {
|
||
client = await this.connect()
|
||
} catch (error: any) {
|
||
// Never started: the lease doesn't count as one of the range's attempts.
|
||
const ipLimit = /simultaneous IP/i.test(error.message)
|
||
log(`NNTP login failed: ${error.message}; trying again in ${ipLimit ? 5 : 1} min`)
|
||
await api
|
||
.post(`/api/v1/seed/ranges/${lease.id}/release`, {
|
||
error: `NNTP login: ${error.message}`,
|
||
uncounted: true,
|
||
})
|
||
.catch(() => {})
|
||
await this.pause(ipLimit ? 300_000 : 60_000)
|
||
continue
|
||
}
|
||
}
|
||
if (!(await this.workRange(api, client, lease, log))) {
|
||
client.close()
|
||
client = undefined
|
||
}
|
||
}
|
||
} finally {
|
||
client?.close()
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Content inspection and archive walks (CONCEPT.md D41): lease a few jobs,
|
||
* probe each with articles through the seed API, post the results. The
|
||
* indexer applies them; a job left unanswered goes back after 10 minutes.
|
||
*/
|
||
private async inspect(api: SeedApi, slot: number) {
|
||
const log = (message: string) => this.logger.info(`[inspect ${this.child && `${this.child}/`}${slot}] ${message}`)
|
||
const fetchArticle = (id: string) => api.get(`/api/v1/seed/articles?id=${encodeURIComponent(id)}`, 4)
|
||
while (!this.stopping) {
|
||
let jobs: InspectJob[] = []
|
||
try {
|
||
jobs = ((await api.post('/api/v1/seed/inspect/lease', { max: 2 })) as { jobs: InspectJob[] } | null)?.jobs ?? []
|
||
} catch (error: any) {
|
||
if (error instanceof Unauthorized) throw error
|
||
log(`Lease failed: ${error.message}`)
|
||
}
|
||
if (jobs.length === 0) {
|
||
if (this.once) return
|
||
await this.pause(30_000)
|
||
continue
|
||
}
|
||
for (const job of jobs) {
|
||
const ref = { queue: job.queue, jobId: job.jobId, token: job.token }
|
||
if (this.stopping) {
|
||
await api.post('/api/v1/seed/inspect/release', ref).catch(() => {})
|
||
continue
|
||
}
|
||
try {
|
||
const result =
|
||
job.task.kind === 'inspect'
|
||
? await probeInspection(job.task, fetchArticle, (message) => log(message))
|
||
: await probeWalk(job.task, fetchArticle)
|
||
await api.post('/api/v1/seed/inspect/done', { ...ref, result: JSON.stringify(result) })
|
||
} catch (error: any) {
|
||
if (error instanceof Unauthorized) throw error
|
||
// The job's lock expired and it went to another node: nothing to report.
|
||
if (error instanceof LeaseLost) continue
|
||
log(`${job.task.kind === 'inspect' ? `"${job.task.name}"` : `walk of release ${job.task.releaseId}`} failed: ${error.message}`)
|
||
await api.post('/api/v1/seed/inspect/failed', { ...ref, error: error.message }).catch(() => {})
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
private connect() {
|
||
return XoverClient.connect({
|
||
host: nntpConfig.host,
|
||
port: nntpConfig.port,
|
||
secure: nntpConfig.secure,
|
||
servername: nntpConfig.tlsServername,
|
||
user: nntpConfig.user,
|
||
password: nntpConfig.password?.release(),
|
||
timeoutMs: nntpConfig.commandTimeoutMs,
|
||
compress: true,
|
||
})
|
||
}
|
||
|
||
/** The next range, or null when there is none (or the API can't be reached; logged). */
|
||
private async lease(api: SeedApi, log: (message: string) => void): Promise<Lease | null> {
|
||
try {
|
||
return ((await api.post('/api/v1/seed/lease')) as { range: Lease } | null)?.range ?? null
|
||
} catch (error: any) {
|
||
if (error instanceof Unauthorized) throw error
|
||
log(`Lease failed: ${error.message}`)
|
||
return null
|
||
}
|
||
}
|
||
|
||
/** Works one leased range; false when the NNTP connection should be replaced. */
|
||
private async workRange(
|
||
api: SeedApi,
|
||
client: OverviewSource,
|
||
lease: Lease,
|
||
log: (message: string) => void
|
||
): Promise<boolean> {
|
||
log(`${lease.group} ${lease.start}–${lease.end}`)
|
||
const started = Date.now()
|
||
let headers = 0
|
||
let chain: Promise<void> = Promise.resolve()
|
||
const posting = new Set<Promise<void>>()
|
||
// Throws once a post has failed, so the range stops fetching.
|
||
const failedSoFar = () => Promise.race([chain, Promise.resolve()])
|
||
try {
|
||
await runRange({
|
||
client,
|
||
group: lease.group,
|
||
start: lease.start,
|
||
end: lease.end,
|
||
chunk: this.chunk,
|
||
submitEvery: this.submitEvery,
|
||
isDeferred: (header: OverviewHeader) =>
|
||
indexerConfig.deferBodyMessageIds.some((re) => re.test(header['message-id'] ?? '')),
|
||
// Batches are posted in order, one after the other, while fetching goes on; with two
|
||
// waiting, fetching waits too. A failed post stops the chain, so no batch is stored
|
||
// past a missing one (the range would resume beyond it).
|
||
sink: async (batch: SeedBatch) => {
|
||
if (this.stopping) throw new Stopping()
|
||
while (posting.size >= 2) await Promise.race(posting)
|
||
const post = chain.then(async () => {
|
||
await api.post(`/api/v1/seed/ranges/${lease.id}/batches`, batch)
|
||
headers += batch.headers
|
||
})
|
||
chain = post
|
||
posting.add(post)
|
||
post.catch(() => {}).finally(() => posting.delete(post))
|
||
await failedSoFar()
|
||
},
|
||
})
|
||
await chain
|
||
await api.post(`/api/v1/seed/ranges/${lease.id}/done`)
|
||
const seconds = (Date.now() - started) / 1000
|
||
log(
|
||
`done: ${headers.toLocaleString('en')} headers in ${seconds.toFixed(0)} s (${Math.round(headers / seconds).toLocaleString('en')}/s)`
|
||
)
|
||
return true
|
||
} catch (error: any) {
|
||
if (error instanceof Unauthorized) throw error
|
||
// An XOVER may still be in flight: the connection is replaced either way.
|
||
if (error instanceof LeaseLost) {
|
||
log(`Range ${lease.id} passed to another node; dropping it.`)
|
||
return false
|
||
}
|
||
const reason = error instanceof Stopping ? 'node stopping' : error.message
|
||
if (!(error instanceof Stopping)) log(`Range ${lease.id} failed: ${reason}`)
|
||
// Let posts in flight finish (or fail) first, so the range resumes after what was stored.
|
||
await chain.catch(() => {})
|
||
await api.post(`/api/v1/seed/ranges/${lease.id}/release`, { error: reason }).catch(() => {})
|
||
if (!this.stopping) await this.pause(30_000)
|
||
return false
|
||
}
|
||
}
|
||
|
||
/** Sleeps, but wakes up early when the node is stopping. */
|
||
private async pause(ms: number) {
|
||
for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)
|
||
}
|
||
}
|
||
|
||
class Unauthorized extends Error {}
|
||
|
||
/** The seed API client: JSON over HTTPS, gzip request bodies, retries for network errors and 5xx. */
|
||
class SeedApi {
|
||
constructor(
|
||
private base: string,
|
||
private token: string,
|
||
private warn: (message: string) => void
|
||
) {}
|
||
|
||
async post(path: string, body?: unknown): Promise<unknown> {
|
||
const payload = body === undefined ? undefined : gzipSync(JSON.stringify(body))
|
||
const result = await this.request('POST', path, payload)
|
||
if (result === null) return null
|
||
const text = result.toString('utf8')
|
||
return text ? JSON.parse(text) : null
|
||
}
|
||
|
||
/** A binary GET (a relay chunk, an article). */
|
||
async get(path: string, attempts = 14): Promise<Buffer> {
|
||
return (await this.request('GET', path, undefined, attempts)) ?? Buffer.alloc(0)
|
||
}
|
||
|
||
private async request(method: string, path: string, payload?: Buffer, attempts = 14): Promise<Buffer | null> {
|
||
for (let attempt = 1; ; attempt++) {
|
||
let status = 0
|
||
let data = Buffer.alloc(0)
|
||
try {
|
||
const response = await fetch(this.base + path, {
|
||
method,
|
||
headers: {
|
||
authorization: `Bearer ${this.token}`,
|
||
...(payload ? { 'content-type': 'application/json', 'content-encoding': 'gzip' } : {}),
|
||
},
|
||
body: payload,
|
||
signal: AbortSignal.timeout(10 * 60 * 1000),
|
||
})
|
||
status = response.status
|
||
data = Buffer.from(await response.arrayBuffer())
|
||
} catch (error: any) {
|
||
data = Buffer.from(error.message)
|
||
}
|
||
const text = () => data.toString('utf8').slice(0, 300)
|
||
if (status === 204) return null
|
||
if (status >= 200 && status < 300) return data
|
||
if (status === 401) throw new Unauthorized('The seed API rejected this node token')
|
||
if (status === 409) throw new LeaseLost(text())
|
||
// 404 comes from the proxy while the seed API restarts (a deploy): this API has no 404s of its own.
|
||
if (status >= 400 && status < 500 && status !== 404)
|
||
throw new Error(`Seed API ${path}: ${status} ${text()}`)
|
||
// Network errors, 404 and 5xx: storing a batch twice is harmless, so try again.
|
||
// ~20 minutes in all, longer than a lease lasts: a restart or maintenance of the API is waited out.
|
||
if (attempt >= attempts) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text()}`)
|
||
const wait = Math.min(120_000, 2000 * 2 ** (attempt - 1))
|
||
this.warn(`Seed API ${path}: ${status || text()}; retrying in ${wait / 1000} s`)
|
||
await sleep(wait)
|
||
}
|
||
}
|
||
}
|
||
|
||
/** A range's overviews through the seed API's relay (CONCEPT.md D38): no news server account on the node. */
|
||
class RelaySource implements OverviewSource {
|
||
constructor(
|
||
private api: SeedApi,
|
||
private rangeId: number
|
||
) {}
|
||
|
||
async group() {}
|
||
|
||
async xover(from: number, to: number): Promise<Overview[]> {
|
||
const raw = await this.api.get(
|
||
`/api/v1/seed/ranges/${this.rangeId}/chunk?from=${from}&to=${to}`
|
||
)
|
||
return raw.length > 0 ? parseOverviews(unzipSync(raw).toString('utf8')) : []
|
||
}
|
||
}
|