The pending ladder in the prober: unnamed releases probed cheapest first, signatures learned, the named moved into the catalog

pending_ladder.ts claims the oldest waiting pending releases' next target
(par2, archive, lone, 7z-tail in that order) as at least PENDING_SHARE of each
claim; a usable name (the largest named file of a par2 set or a listing, or a
lone file's yEnc name, neither packaging nor random) hands the release to the
namer, no name the next target or the end of the climb; climbs count on
pending_signatures per (tool, group, poster) and (tool, group), and a signature
with 100 climbs and 0.5% named or fewer is dropped: its waiting releases are
discarded without a probe, the rules' sample excepted. pending_namer.ts reads
a named release's articles again from the corpus or by XOVER and writes it to
the catalog under the pending key (source weak-stem, the name as inner name)
with its chunk and candidates. Migration 6400; config PENDING_*.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/services/prober.ts

@@ -9,12 +9,16 @@ import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/i
 import { NntpShare } from '#services/nntp_shares'
 import { DiskSpace } from '#services/disk_space'
 import { decidePar2Completeness } from '#services/par2_completeness'
+import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
+import { PendingNamer } from '#services/pending_namer'
 
 interface Candidate {
   id: number
   releaseId: number
   kind: string
   messageId: string
+  /** A pending release's target (the pending ladder), not a catalog candidate: id and releaseId are 0. */
+  pending?: PendingProbe
 }
 
 type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
@@ -34,6 +38,8 @@ const TIERS: { kind: string; unreadable: boolean }[] = [
   { kind: '7z-tail', unreadable: false },
 ]
 const FLUSH_MS = 2000
+/** Between sweeps that discard the waiting releases of dropped signatures. */
+const SWEEP_MS = 5 * 60_000
 
 /**
  * JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
@@ -72,13 +78,19 @@ export class Prober {
 
   /** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
   private static freshStats() {
-    return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
+    return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, pending: 0, discarded: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
   }
+  /** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
+  private namer = new PendingNamer()
+  private sweptAt = 0
 
   async run() {
     // A previous run's claims: one prober at a time.
     const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
     if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
+    const released = await PendingLadder.release()
+    if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
+    const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
     logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
     const flusher = (async () => {
       while (!this.stopped || this.outcomes.length > 0) {
@@ -108,6 +120,7 @@ export class Prober {
       for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !this.stopped; waited += 1000) await sleep(1000)
     }
     await Promise.all(this.lanes)
+    await namer
     await this.share.leave().catch(() => {})
     // What was claimed and not probed goes back.
     for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
@@ -227,6 +240,37 @@ export class Prober {
   private async claimTiers() {
     // The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again.
     if (await this.space.short()) return
+    await this.claimCatalog()
+    await this.claimPending()
+  }
+
+  /** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
+  private async claimPending() {
+    if (Date.now() - this.sweptAt > SWEEP_MS) {
+      this.sweptAt = Date.now()
+      this.stats.discarded += await PendingLadder.discard()
+    }
+    const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
+    const probes = await PendingLadder.claim(want)
+    if (probes.length === 0) return
+    const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
+      probes.map((p) => p.messageId),
+      PROBE_VERSION,
+    ])
+    const known = new Map(cached.map((c: any) => [c.message_id, c]))
+    for (const probe of probes) {
+      const candidate: Candidate = { id: 0, releaseId: 0, kind: probe.kind, messageId: probe.messageId, pending: probe }
+      const hit: any = known.get(probe.messageId)
+      if (hit) {
+        this.stats.cached++
+        this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
+      } else {
+        this.queue.push(candidate)
+      }
+    }
+  }
+
+  private async claimCatalog() {
     for (const { kind, unreadable } of TIERS) {
       const want = this.batch - this.queue.length
       if (want <= 0) return
@@ -277,18 +321,25 @@ export class Prober {
 
   /** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
   private async flush() {
-    const outcomes = this.outcomes.splice(0)
-    if (outcomes.length === 0) return
-    for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
+    const taken = this.outcomes.splice(0)
+    if (taken.length === 0) return
+    for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
+    // The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
+    const pending = taken.filter((o) => o.candidate.pending)
+    const outcomes = taken.filter((o) => !o.candidate.pending)
+    this.stats.pending += pending.filter((o) => o.status !== 'waiting').length
     await db.transaction(async (trx) => {
-      await trx.rawQuery(
-        `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
-        [jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
-      )
+      if (outcomes.length > 0) {
+        await trx.rawQuery(
+          `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
+          [jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
+        )
+      }
+      await PendingLadder.record(pending.map((o) => ({ probe: o.candidate.pending!, status: o.status, inspection: o.inspection })), trx)
       // Final answers read from the news server, kept by message-id for the next catalog (a reset).
       // One per message-id: the same article is a candidate of more than one release, and ON CONFLICT can't take a
       // row twice in one statement.
-      const fresh = [...new Map(outcomes.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
+      const fresh = [...new Map(taken.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
       if (fresh.length > 0) {
         await trx.rawQuery(
           `INSERT INTO probe_cache (message_id, status, result, version)
@@ -365,7 +416,8 @@ export class Prober {
     const { articles, bytes, done, missing, error } = this.stats
     logger.info(
       `Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
-        `${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache. ` +
+        `${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache; ` +
+        `the pending ladder ${this.stats.pending} probed, ${this.stats.discarded} discarded by signature. ` +
         `Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
         `idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
         `claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`