Content inspection and archive walks on seeding nodes (relay)
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51975fe603
commit
2dbca9b152
22 files changed
+1198
-540
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator'
|
||||
import SeedFetchPool from '#services/SeedFetchPool'
|
||||
import RemoteContentService, { JobLost } from '#services/RemoteContentService'
|
||||
import type { SeedBatch } from '#lib/seed/wire'
|
||||
|
||||
/**
|
||||
@@ -71,11 +72,77 @@ export default class SeedController {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/seed/inspect/lease { max } → 200 { jobs } or 204: content
|
||||
* inspection and archive walks for the node (CONCEPT.md D41).
|
||||
*/
|
||||
async inspectLease({ seedNode, request, response }: HttpContext) {
|
||||
const max = Math.min(10, Math.max(1, Number(request.input('max')) || 1))
|
||||
const jobs = await RemoteContentService.lease(seedNode.id, max)
|
||||
return jobs.length > 0 ? { jobs } : response.noContent()
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/seed/inspect/done { queue, jobId, token, result } where
|
||||
* result is the probe as a JSON string (the body parser would turn the
|
||||
* empty strings inside it into nulls).
|
||||
*/
|
||||
async inspectDone({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token, result } = request.only(['queue', 'jobId', 'token', 'result'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string' || typeof result !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token, result: JSON string }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.finish(queue, jobId, token, JSON.parse(result)))
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/inspect/failed { queue, jobId, token, error } */
|
||||
async inspectFailed({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token, error } = request.only(['queue', 'jobId', 'token', 'error'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token, error }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.fail(queue, jobId, token, String(error ?? 'unknown error')))
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/inspect/release { queue, jobId, token }: a stopping node gives a job back. */
|
||||
async inspectRelease({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token } = request.only(['queue', 'jobId', 'token'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.release(queue, jobId, token))
|
||||
}
|
||||
|
||||
/** GET /api/v1/seed/articles?id=<message id> → the raw article body; 410 when the server doesn't have it. */
|
||||
async article({ request, response }: HttpContext) {
|
||||
const id = request.input('id')
|
||||
if (typeof id !== 'string' || id.length === 0 || id.length > 250) return response.badRequest({ error: 'Expected ?id=<message id>' })
|
||||
try {
|
||||
const body = await RemoteContentService.article(id)
|
||||
if (!body) return response.gone({ error: 'No such article' })
|
||||
response.header('content-type', 'application/octet-stream')
|
||||
return response.send(body)
|
||||
} catch (error: any) {
|
||||
return response.badGateway({ error: `NNTP: ${error.message}` })
|
||||
}
|
||||
}
|
||||
|
||||
/** GET /api/v1/seed/status */
|
||||
async status() {
|
||||
return SeedOrchestrator.status()
|
||||
}
|
||||
|
||||
/** 409 when the job's lock is gone (it was given to another node meanwhile). */
|
||||
private async job(response: HttpContext['response'], action: () => Promise<unknown>) {
|
||||
try {
|
||||
await action()
|
||||
return { ok: true }
|
||||
} catch (error) {
|
||||
if (error instanceof JobLost) return response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** 409 when the range was given to another node meanwhile; the node should drop it. */
|
||||
private async leased(response: HttpContext['response'], action: () => Promise<unknown>) {
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user