Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/ArchiveWalkService.ts

@@ -1,13 +1,12 @@
 import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
 import env from '#start/env'
 import NntpService from '#services/NntpService'
 import QueueService from '#services/QueueService'
 import { segmentsOf } from '#services/FileStore'
 import { looksRandom, releaseNameFromContents } from '#services/ArchiveService'
-import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
 import { rarVolume } from '#lib/content/index'
-import { walkRar, type RarInfo, type VolumeReader } from '#lib/rar/index'
+import type { RarEntry, RarInfo } from '#lib/rar/index'
+import { probeWalk, type WalkProbe, type WalkTask } from '#lib/content/probe'
 
 /** Stored per release, as for a first-volume listing (ArchiveService). */
 const MAX_ENTRIES = 500
@@ -55,33 +54,50 @@ class ArchiveWalkService {
     await QueueService.archiveWalkQueue.add('walk', { releaseId }, { jobId: `walk-${releaseId}`, delay: WALK_DELAY_MS })
   }
 
-  async walk(releaseId: number): Promise<WalkOutcome> {
-    const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'size', 'contents', 'encrypted').first()
-    if (!release) return { releaseId, skipped: 'missing' }
+  /**
+   * The database half before a walk (D41): the release's volumes, or the
+   * outcome when there's nothing to walk.
+   */
+  async prepare(releaseId: number): Promise<{ task: WalkTask } | { outcome: WalkOutcome }> {
+    const release = await db.from('releases').where('id', releaseId).select('id', 'size', 'contents').first()
+    if (!release) return { outcome: { releaseId, skipped: 'missing' } }
     const before = Array.isArray(release.contents) ? release.contents : []
     const listed = before.reduce((sum: number, e: any) => sum + Number(e.size ?? 0), 0)
-    if (!this.worthWalking(before.length, listed, Number(release.size))) return this.done(releaseId, { releaseId, skipped: 'covered' })
-
+    if (!this.worthWalking(before.length, listed, Number(release.size))) {
+      return { outcome: await this.done(releaseId, { releaseId, skipped: 'covered' }) }
+    }
     const volumes = await this.volumes(releaseId)
-    if (!volumes) return this.done(releaseId, { releaseId, skipped: 'volumes' })
+    if (!volumes) return { outcome: await this.done(releaseId, { releaseId, skipped: 'volumes' }) }
+    return {
+      task: {
+        kind: 'walk',
+        releaseId,
+        volumes: volumes.map((file) => segmentsOf(file).map(({ number, id }) => ({ number, id }))),
+        maxSegments: this.maxSegments,
+      },
+    }
+  }
 
-    const reader = new SegmentReader(volumes, this.maxSegments)
-    const walked = await walkRar(reader)
+  /** The database half after a walk: the fuller listing, the password flag and a better name. */
+  async apply(releaseId: number, walked: WalkProbe): Promise<WalkOutcome> {
+    const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'contents', 'encrypted').first()
+    if (!release) return { releaseId, skipped: 'missing' }
+    const before = Array.isArray(release.contents) ? release.contents : []
     const outcome: WalkOutcome = {
       releaseId,
       before: before.length,
       entries: walked.entries.length,
       complete: walked.complete,
       stopped: walked.stopped,
-      segments: reader.fetched,
+      segments: walked.segments,
     }
     if (walked.entries.length > before.length) {
-      const contents = walked.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry)
+      const contents = walked.entries.slice(0, MAX_ENTRIES)
       const update: Record<string, unknown> = {
         contents: JSON.stringify(contents),
         encrypted: Boolean(release.encrypted) || walked.headersEncrypted || walked.entries.some((e) => e.encrypted),
       }
-      const better = releaseNameFromContents({ entries: walked.entries })
+      const better = releaseNameFromContents({ entries: walked.entries as RarEntry[] })
       // The rule of ArchiveService.record: random and post-subject names give way, a spot's title stays.
       if (better && better !== release.name && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post')) {
         update.name = better
@@ -92,6 +108,13 @@ class ArchiveWalkService {
     return this.done(releaseId, outcome)
   }
 
+  /** Walks a release here, on the inspection pool's connections when inspection isn't waiting for one. */
+  async walk(releaseId: number): Promise<WalkOutcome> {
+    const prepared = await this.prepare(releaseId)
+    if ('outcome' in prepared) return prepared.outcome
+    return this.apply(releaseId, await probeWalk(prepared.task, fetchLow))
+  }
+
   private async done(releaseId: number, outcome: WalkOutcome) {
     await db.from('releases').where('id', releaseId).update({ contents_walked_at: new Date() })
     return outcome
@@ -120,82 +143,6 @@ class ArchiveWalkService {
   }
 }
 
-/**
- * Reads volumes a segment at a time, finding the segment for an offset from
- * the part size (yEnc posters use one size per set) and correcting from the
- * `=ypart` offsets when a guess is off.
- */
-export class SegmentReader implements VolumeReader {
-  fetched = 0
-  private partSize = 0
-  private readonly sizes = new Map<number, number>()
-  private readonly cache = new Map<string, DecodedSegment>()
-
-  constructor(
-    private readonly files: any[],
-    private readonly maxSegments: number,
-    private readonly fetchBody: (messageId: string) => Promise<Buffer> = fetchLow
-  ) {}
-
-  get volumes() {
-    return this.files.length
-  }
-
-  size(v: number) {
-    return this.sizes.get(v) ?? null
-  }
-
-  async read(v: number, offset: number, more: boolean): Promise<Buffer | null> {
-    const count = segmentsOf(this.files[v]).length
-    let index = this.partSize ? Math.min(count - 1, Math.floor(offset / this.partSize)) : 0
-    for (let tries = 0; tries < 4; tries++) {
-      const segment = await this.segment(v, index)
-      if (!segment) return null
-      const end = segment.offset + segment.data.length
-      if (offset >= segment.offset && offset < end) {
-        const data = segment.data.subarray(offset - segment.offset)
-        if (!more || index + 1 >= count) return data
-        const next = await this.segment(v, index + 1)
-        return next ? Buffer.concat([data, next.data]) : null
-      }
-      const step = Math.floor((offset - segment.offset) / Math.max(1, segment.data.length))
-      const moved = Math.max(0, Math.min(count - 1, index + (step === 0 ? (offset < segment.offset ? -1 : 1) : step)))
-      if (moved === index) return null
-      index = moved
-    }
-    return null
-  }
-
-  private async segment(v: number, index: number): Promise<DecodedSegment | null> {
-    const key = `${v}:${index}`
-    const cached = this.cache.get(key)
-    if (cached) return cached
-    const part = segmentsOf(this.files[v])[index]
-    if (!part?.id || this.fetched >= this.maxSegments) return null
-    this.fetched++
-    let body: Buffer
-    try {
-      body = await this.fetchBody(part.id)
-    } catch (error: any) {
-      logger.debug(`Archive walk: segment ${index} of file ${this.files[v].id} unavailable: ${error.message}`)
-      return null
-    }
-    let segment: DecodedSegment
-    try {
-      segment = decodeSegment(body)
-    } catch (error) {
-      if (!(error instanceof YencError) || error instanceof YencCrcError) return null
-      segment = decodeSegment(body, { lenientSize: true })
-    }
-    this.sizes.set(v, segment.meta.begin.size)
-    if (!this.partSize && segment.meta.part) this.partSize = segment.meta.part.end - segment.meta.part.begin + 1
-    // Walks move forward: the last few segments are enough.
-    if (this.cache.size >= 4) this.cache.delete(this.cache.keys().next().value!)
-    this.cache.set(key, segment)
-    return segment
-  }
-}
-
 /** One article body on a pooled connection, behind inspection's waiting jobs. */
 async function fetchLow(messageId: string): Promise<Buffer> {
   const conn = await NntpService.acquire(true)