Content inspection and archive walks on seeding nodes (relay)
Test and deploy / test (push) Successful in 1m49s
Test and deploy / deploy (push) Successful in 2m15s

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 11:17:02 +02:00
1 parent 51975fe603
commit 2dbca9b152
22 files changed
+1198 -540

No files matched your search

+67
View File
@@ -1,6 +1,7 @@
import type { HttpContext } from '@adonisjs/core/http'
import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator'
import SeedFetchPool from '#services/SeedFetchPool'
import RemoteContentService, { JobLost } from '#services/RemoteContentService'
import type { SeedBatch } from '#lib/seed/wire'
/**
@@ -71,11 +72,77 @@ export default class SeedController {
})
}
/**
* POST /api/v1/seed/inspect/lease { max } → 200 { jobs } or 204: content
* inspection and archive walks for the node (CONCEPT.md D41).
*/
async inspectLease({ seedNode, request, response }: HttpContext) {
const max = Math.min(10, Math.max(1, Number(request.input('max')) || 1))
const jobs = await RemoteContentService.lease(seedNode.id, max)
return jobs.length > 0 ? { jobs } : response.noContent()
}
/**
* POST /api/v1/seed/inspect/done { queue, jobId, token, result } where
* result is the probe as a JSON string (the body parser would turn the
* empty strings inside it into nulls).
*/
async inspectDone({ request, response }: HttpContext) {
const { queue, jobId, token, result } = request.only(['queue', 'jobId', 'token', 'result'])
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string' || typeof result !== 'string') {
return response.badRequest({ error: 'Expected { queue, jobId, token, result: JSON string }' })
}
return this.job(response, () => RemoteContentService.finish(queue, jobId, token, JSON.parse(result)))
}
/** POST /api/v1/seed/inspect/failed { queue, jobId, token, error } */
async inspectFailed({ request, response }: HttpContext) {
const { queue, jobId, token, error } = request.only(['queue', 'jobId', 'token', 'error'])
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
return response.badRequest({ error: 'Expected { queue, jobId, token, error }' })
}
return this.job(response, () => RemoteContentService.fail(queue, jobId, token, String(error ?? 'unknown error')))
}
/** POST /api/v1/seed/inspect/release { queue, jobId, token }: a stopping node gives a job back. */
async inspectRelease({ request, response }: HttpContext) {
const { queue, jobId, token } = request.only(['queue', 'jobId', 'token'])
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
return response.badRequest({ error: 'Expected { queue, jobId, token }' })
}
return this.job(response, () => RemoteContentService.release(queue, jobId, token))
}
/** GET /api/v1/seed/articles?id=<message id> → the raw article body; 410 when the server doesn't have it. */
async article({ request, response }: HttpContext) {
const id = request.input('id')
if (typeof id !== 'string' || id.length === 0 || id.length > 250) return response.badRequest({ error: 'Expected ?id=<message id>' })
try {
const body = await RemoteContentService.article(id)
if (!body) return response.gone({ error: 'No such article' })
response.header('content-type', 'application/octet-stream')
return response.send(body)
} catch (error: any) {
return response.badGateway({ error: `NNTP: ${error.message}` })
}
}
/** GET /api/v1/seed/status */
async status() {
return SeedOrchestrator.status()
}
/** 409 when the job's lock is gone (it was given to another node meanwhile). */
private async job(response: HttpContext['response'], action: () => Promise<unknown>) {
try {
await action()
return { ok: true }
} catch (error) {
if (error instanceof JobLost) return response.conflict({ error: error.message })
throw error
}
}
/** 409 when the range was given to another node meanwhile; the node should drop it. */
private async leased(response: HttpContext['response'], action: () => Promise<unknown>) {
try {