Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/SeedNode.ts

@@ -8,10 +8,18 @@ import { XoverClient, parseOverviews, type Overview } from '#lib/nntp/xover_clie
 import { runRange, type OverviewSource } from '#lib/seed/runner'
 import type { SeedBatch } from '#lib/seed/wire'
 import type { OverviewHeader } from '#lib/ingest/segment'
+import { probeInspection, probeWalk, type ContentTask } from '#lib/content/probe'
 
 class LeaseLost extends Error {}
 class Stopping extends Error {}
 
+interface InspectJob {
+  queue: string
+  jobId: string
+  token: string
+  task: ContentTask
+}
+
 interface Lease {
   id: number
   group: string
@@ -45,11 +53,18 @@ export default class SeedNode extends BaseCommand {
 
   @flags.number({
     description:
-      'Ranges worked on at once, one NNTP connection each (default: SEED_CONNECTIONS or 1)',
-    default: Number(process.env.SEED_CONNECTIONS) || 1,
+      'Ranges worked on at once, one NNTP connection each; 0 for an inspection-only node (default: SEED_CONNECTIONS or 1)',
+    default: process.env.SEED_CONNECTIONS ? Number(process.env.SEED_CONNECTIONS) : 1,
   })
   declare connections: number
 
+  @flags.number({
+    description:
+      'Content inspections and archive walks worked on at once, articles through the seed API (default: SEED_INSPECTORS or 0)',
+    default: Number(process.env.SEED_INSPECTORS) || 0,
+  })
+  declare inspectors: number
+
   @flags.number({ description: 'Articles per XOVER', default: 20_000 })
   declare chunk: number
 
@@ -85,9 +100,13 @@ export default class SeedNode extends BaseCommand {
 
     const api = new SeedApi(base, token, (message) => this.logger.warning(message))
     this.logger.info(
-      `Seeding node for ${base}: ${this.connections} range(s) at once, ${this.fetch === 'direct' ? 'own NNTP connections' : 'headers through the seed API'}.`
+      `Seeding node for ${base}: ${this.connections} range(s) at once, ${this.fetch === 'direct' ? 'own NNTP connections' : 'headers through the seed API'}` +
+        (this.inspectors > 0 ? `; ${this.inspectors} inspection(s) at once, articles through the seed API.` : '.')
     )
-    await Promise.all(Array.from({ length: this.connections }, (_, i) => this.work(api, i + 1)))
+    await Promise.all([
+      ...Array.from({ length: this.connections }, (_, i) => this.work(api, i + 1)),
+      ...Array.from({ length: this.inspectors }, (_, i) => this.inspect(api, i + 1)),
+    ])
     this.logger.info('Seeding node stopped.')
   }
 
@@ -137,6 +156,50 @@ export default class SeedNode extends BaseCommand {
     }
   }
 
+  /**
+   * Content inspection and archive walks (CONCEPT.md D41): lease a few jobs,
+   * probe each with articles through the seed API, post the results. The
+   * indexer applies them; a job left unanswered goes back after 10 minutes.
+   */
+  private async inspect(api: SeedApi, slot: number) {
+    const log = (message: string) => this.logger.info(`[inspect ${slot}] ${message}`)
+    const fetchArticle = (id: string) => api.get(`/api/v1/seed/articles?id=${encodeURIComponent(id)}`, 4)
+    while (!this.stopping) {
+      let jobs: InspectJob[] = []
+      try {
+        jobs = ((await api.post('/api/v1/seed/inspect/lease', { max: 2 })) as { jobs: InspectJob[] } | null)?.jobs ?? []
+      } catch (error: any) {
+        if (error instanceof Unauthorized) throw error
+        log(`Lease failed: ${error.message}`)
+      }
+      if (jobs.length === 0) {
+        if (this.once) return
+        await this.pause(30_000)
+        continue
+      }
+      for (const job of jobs) {
+        const ref = { queue: job.queue, jobId: job.jobId, token: job.token }
+        if (this.stopping) {
+          await api.post('/api/v1/seed/inspect/release', ref).catch(() => {})
+          continue
+        }
+        try {
+          const result =
+            job.task.kind === 'inspect'
+              ? await probeInspection(job.task, fetchArticle, (message) => log(message))
+              : await probeWalk(job.task, fetchArticle)
+          await api.post('/api/v1/seed/inspect/done', { ...ref, result: JSON.stringify(result) })
+        } catch (error: any) {
+          if (error instanceof Unauthorized) throw error
+          // The job's lock expired and it went to another node: nothing to report.
+          if (error instanceof LeaseLost) continue
+          log(`${job.task.kind === 'inspect' ? `"${job.task.name}"` : `walk of release ${job.task.releaseId}`} failed: ${error.message}`)
+          await api.post('/api/v1/seed/inspect/failed', { ...ref, error: error.message }).catch(() => {})
+        }
+      }
+    }
+  }
+
   private connect() {
     return XoverClient.connect({
       host: nntpConfig.host,
@@ -249,12 +312,12 @@ class SeedApi {
     return text ? JSON.parse(text) : null
   }
 
-  /** A binary GET (a relay chunk). */
-  async get(path: string): Promise<Buffer> {
-    return (await this.request('GET', path)) ?? Buffer.alloc(0)
+  /** A binary GET (a relay chunk, an article). */
+  async get(path: string, attempts = 14): Promise<Buffer> {
+    return (await this.request('GET', path, undefined, attempts)) ?? Buffer.alloc(0)
   }
 
-  private async request(method: string, path: string, payload?: Buffer): Promise<Buffer | null> {
+  private async request(method: string, path: string, payload?: Buffer, attempts = 14): Promise<Buffer | null> {
     for (let attempt = 1; ; attempt++) {
       let status = 0
       let data = Buffer.alloc(0)
@@ -283,7 +346,7 @@ class SeedApi {
         throw new Error(`Seed API ${path}: ${status} ${text()}`)
       // Network errors, 404 and 5xx: storing a batch twice is harmless, so try again.
       // ~20 minutes in all, longer than a lease lasts: a restart or maintenance of the API is waited out.
-      if (attempt >= 14) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text()}`)
+      if (attempt >= attempts) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text()}`)
       const wait = Math.min(120_000, 2000 * 2 ** (attempt - 1))
       this.warn(`Seed API ${path}: ${status || text()}; retrying in ${wait / 1000} s`)
       await sleep(wait)