The pending ladder in the prober: unnamed releases probed cheapest first, signatures learned, the named moved into the catalog

pending_ladder.ts claims the oldest waiting pending releases' next target
(par2, archive, lone, 7z-tail in that order) as at least PENDING_SHARE of each
claim; a usable name (the largest named file of a par2 set or a listing, or a
lone file's yEnc name, neither packaging nor random) hands the release to the
namer, no name the next target or the end of the climb; climbs count on
pending_signatures per (tool, group, poster) and (tool, group), and a signature
with 100 climbs and 0.5% named or fewer is dropped: its waiting releases are
discarded without a probe, the rules' sample excepted. pending_namer.ts reads
a named release's articles again from the corpus or by XOVER and writes it to
the catalog under the pending key (source weak-stem, the name as inner name)
with its chunk and candidates. Migration 6400; config PENDING_*.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -147,7 +147,7 @@ class BatchStore {
   }
 
   /** The range's block; a range without one (no planner yet) is a block of its own, leased as the range is. */
-  private async blockOf(trx: TransactionClientContract, range: any): Promise<number> {
+  async blockOf(trx: TransactionClientContract, range: any): Promise<number> {
     if (range.block_id) return Number(range.block_id)
     const [block] = await trx
       .table('blocks')
@@ -204,21 +204,22 @@ class BatchStore {
           bytes: r.bytes,
           partial: r.partial === true,
           finished: r.finished !== false,
+          file_total: r.fileTotal,
           tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
           candidates: probes,
         }
       })
       const { rows: done } = await trx.rawQuery(
         `INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
-                                       files, segments, bytes, partial, finished, tool, candidates)
+                                       files, segments, bytes, partial, finished, file_total, tool, candidates)
          SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
-                v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.tool, v.candidates
+                v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates
          FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
-                                               files int, segments bigint, bytes bigint, partial boolean, finished boolean, tool text, candidates jsonb)
+                                               files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb)
          ON CONFLICT (release_key) DO UPDATE SET
            files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
            last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
-           partial = excluded.partial, finished = excluded.finished,
+           partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
            candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END
          WHERE NOT pending_releases.finished
          RETURNING release_key`,

app/services/corpus.ts

@@ -42,13 +42,13 @@ class Corpus {
     return coordinatorConfig.corpus.dir
   }
 
-  private rangeDir(range: CorpusRange) {
-    return join(this.dir!, range.group, `${range.start}-${range.end}`)
+  private rangeDir(range: CorpusRange, dir = this.dir!) {
+    return join(dir, range.group, `${range.start}-${range.end}`)
   }
 
-  private path(range: CorpusRange, span: ChunkSpan) {
+  private path(range: CorpusRange, span: ChunkSpan, dir = this.dir!) {
     const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
-    return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
+    return join(this.rangeDir(range, dir), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
   }
 
   /** Whether every chunk of the range is stored (its range.json). */
@@ -69,10 +69,10 @@ class Corpus {
     )
   }
 
-  /** A stored chunk, or null. */
-  async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
-    if (!this.dir) return null
-    return readFile(this.path(range, span)).catch(() => null)
+  /** A stored chunk, or null (`dir`: another corpus than CORPUS_DIR's, the namer's tests). */
+  async read(range: CorpusRange, span: ChunkSpan, dir = this.dir): Promise<Buffer | null> {
+    if (!dir) return null
+    return readFile(this.path(range, span, dir)).catch(() => null)
   }
 
   /**

app/services/pending_ladder.ts

@@ -0,0 +1,230 @@
+import db from '@adonisjs/lucid/services/db'
+import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
+import coordinatorConfig from '#config/coordinator'
+import { explored } from '#lib/rules/index'
+import { usableName, type Inspection } from '#lib/probe/index'
+import RuleSets from '#services/rule_sets'
+
+/** pending_releases.status (migration 6400). */
+export const PENDING = {
+  waiting: 0,
+  named: 1,
+  discarded: 2,
+  exhausted: 3,
+  naming: 4,
+  taken: 9,
+} as const
+
+/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
+export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
+
+export interface PendingTarget {
+  kind: string
+  messageId: string
+  file: string
+}
+
+/** One probe the ladder hands the prober: the next target of a pending release. */
+export interface PendingProbe {
+  /** pending_releases.release_key as hex. */
+  releaseKey: string
+  kind: string
+  messageId: string
+  file: string
+  tool: string
+  groupId: number
+  poster: string
+  /** Targets probed before this one. */
+  tries: number
+  targets: number
+}
+
+export type PendingOutcome = {
+  probe: PendingProbe
+  status: 'done' | 'missing' | 'error' | 'waiting'
+  inspection?: Inspection
+}
+
+export function byEffort(targets: PendingTarget[]): PendingTarget[] {
+  const rank = (kind: string) => {
+    const i = (EFFORT as readonly string[]).indexOf(kind)
+    return i < 0 ? EFFORT.length : i
+  }
+  return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
+}
+
+/**
+ * The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
+ * handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
+ * every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
+ * signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
+ * a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
+ */
+class PendingLadder {
+  /** A previous prober's claims back in the queue. */
+  async release(): Promise<number> {
+    const { rowCount } = await db.rawQuery(
+      `UPDATE pending_releases SET status = ? WHERE status = ?`,
+      [PENDING.waiting, PENDING.taken]
+    )
+    return rowCount ?? 0
+  }
+
+  /** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
+  async claim(n: number): Promise<PendingProbe[]> {
+    if (n <= 0) return []
+    const { rows } = await db.rawQuery(
+      `UPDATE pending_releases p SET status = ? FROM (
+         SELECT release_key FROM pending_releases
+         WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
+         ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
+       WHERE p.release_key = c.release_key
+       RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
+      [PENDING.taken, PENDING.waiting, n]
+    )
+    return rows.map((r: any) => {
+      const targets = byEffort(r.candidates as PendingTarget[])
+      const tries = Number(r.tries)
+      const target = targets[Math.min(tries, targets.length - 1)]
+      return {
+        releaseKey: r.release_key,
+        kind: target.kind,
+        messageId: target.messageId,
+        file: target.file,
+        tool: r.tool,
+        groupId: Number(r.group_id),
+        poster: r.poster,
+        tries,
+        targets: targets.length,
+      }
+    })
+  }
+
+  /**
+   * What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
+   * end of the ladder (status 3); the finished climbs counted on their signatures.
+   */
+  async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
+    if (outcomes.length === 0) return
+    const rows = outcomes.map((o) => {
+      const p = o.probe
+      if (o.status === 'waiting') {
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.waiting,
+          tries: p.tries,
+          named: null,
+          inspection: null,
+          final: null,
+        }
+      }
+      const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
+      const tries = p.tries + 1
+      const inspection = {
+        kind: p.kind,
+        status: o.status,
+        result: o.inspection ? { ...o.inspection, par2: undefined } : null,
+      }
+      if (name)
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.naming,
+          tries: 0,
+          named: name,
+          inspection,
+          final: 'named',
+        }
+      if (tries >= p.targets)
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.exhausted,
+          tries,
+          named: null,
+          inspection,
+          final: 'exhausted',
+        }
+      return {
+        release_key: p.releaseKey,
+        status: PENDING.waiting,
+        tries,
+        named: null,
+        inspection,
+        final: null,
+      }
+    })
+    await trx.rawQuery(
+      `UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
+         inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
+         probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
+       FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
+       WHERE p.release_key = decode(v.release_key, 'hex')`,
+      [JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
+    )
+    // The signatures: the poster's own and the tool's and group's together.
+    const counts = new Map<
+      string,
+      { tool: string; group_id: number; poster: string; probed: number; named: number }
+    >()
+    for (const [i, o] of outcomes.entries()) {
+      const final = rows[i].final
+      if (!final) continue
+      for (const poster of [o.probe.poster, '']) {
+        const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
+        const entry = counts.get(key) ?? {
+          tool: o.probe.tool,
+          group_id: o.probe.groupId,
+          poster,
+          probed: 0,
+          named: 0,
+        }
+        entry.probed++
+        if (final === 'named') entry.named++
+        counts.set(key, entry)
+      }
+    }
+    if (counts.size === 0) return
+    const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
+    await trx.rawQuery(
+      `INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
+       SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
+       ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
+      [JSON.stringify([...counts.values()])]
+    )
+    await trx.rawQuery(
+      `UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
+       FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
+       WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
+      [
+        pendingMinProbes,
+        pendingMaxNamed,
+        JSON.stringify(
+          [...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
+        ),
+      ]
+    )
+  }
+
+  /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
+  async discard(limit = 10_000): Promise<number> {
+    const { samplePercent } = await RuleSets.current()
+    const { rows } = await db.rawQuery(
+      `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
+       WHERE p.status = ? AND EXISTS (
+         SELECT 1 FROM pending_signatures s
+         WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
+       LIMIT ?`,
+      [PENDING.waiting, limit]
+    )
+    const keys = rows
+      .map((r: any) => r.release_key as string)
+      .filter((key: string) => !explored(key, samplePercent))
+    if (keys.length === 0) return 0
+    const { rowCount } = await db.rawQuery(
+      `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
+      [PENDING.discarded, PENDING.waiting, keys]
+    )
+    return rowCount ?? 0
+  }
+}
+
+export default new PendingLadder()

app/services/pending_namer.ts

@@ -0,0 +1,222 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { promisify } from 'node:util'
+import { unzip } from 'node:zlib'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { RangeIndexer, type ReleaseChunk } from '#lib/edge/index'
+import { identityKey } from '#lib/release/keys'
+import BatchStore from '#services/batch_store'
+import Corpus from '#services/corpus'
+import RuleSets from '#services/rule_sets'
+import { PENDING } from '#services/pending_ladder'
+
+const inflate = promisify(unzip)
+
+interface NamedRow {
+  release_key: Buffer
+  identity_key: Buffer
+  group_id: number
+  named: string
+  first_article: number
+  last_article: number
+  file_total: number | null
+  tries: number
+}
+
+/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
+const MAX_MOVES = 3
+
+/**
+ * The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
+ * read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
+ * connection of its own, opened for it and closed after), through the TypeScript indexer, and is written as a
+ * release with the name the probe found (source weak-stem, the name as inner name), its chunk and its probe
+ * candidates, which the prober answers from probe_cache. Runs inside the prober.
+ */
+export class PendingNamer {
+  private client: XoverClient | null = null
+  private usedAt = 0
+
+  constructor(private corpusDir: string | undefined = coordinatorConfig.corpus.dir) {}
+
+  async run(stopped: () => boolean) {
+    while (!stopped()) {
+      let rows: NamedRow[] = []
+      try {
+        rows = await this.take(20)
+        for (const row of rows) {
+          if (stopped()) break
+          try {
+            await this.move(row)
+          } catch (error: any) {
+            logger.warn(
+              `Naming ${row.release_key.toString('hex')} "${row.named}" failed: ${error.message}`
+            )
+            await db.rawQuery(
+              `UPDATE pending_releases SET tries = tries + 1, probed_at = now() WHERE release_key = ?`,
+              [row.release_key]
+            )
+          }
+        }
+      } catch (error: any) {
+        logger.error(`Naming: ${error.message}`)
+      }
+      if (this.client && Date.now() - this.usedAt > coordinatorConfig.nntpIdleCloseMs) {
+        this.client.close()
+        this.client = null
+      }
+      if (rows.length === 0)
+        for (let waited = 0; waited < 30_000 && !stopped(); waited += 1000) await sleep(1000)
+    }
+    this.client?.close()
+    this.client = null
+  }
+
+  /** Named releases awaiting their move, the longest waiting first; those failed MAX_MOVES times stay as they are. */
+  async take(n: number): Promise<NamedRow[]> {
+    const { rows } = await db.rawQuery(
+      `SELECT release_key, identity_key, group_id, named, first_article, last_article, file_total, tries
+       FROM pending_releases WHERE status = ? AND tries < ? ORDER BY probed_at LIMIT ?`,
+      [PENDING.naming, MAX_MOVES, n]
+    )
+    return rows.map((r: any) => ({
+      ...r,
+      group_id: Number(r.group_id),
+      first_article: Number(r.first_article),
+      last_article: Number(r.last_article),
+      tries: Number(r.tries),
+    }))
+  }
+
+  /** One named release into the catalog. */
+  async move(row: NamedRow): Promise<void> {
+    const group = await db.from('groups').where('id', row.group_id).firstOrFail()
+    const range = await db
+      .from('ranges')
+      .where('group_id', row.group_id)
+      .where('start_id', '<=', row.first_article)
+      .where('end_id', '>=', row.first_article)
+      .orderBy('id')
+      .first()
+    if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
+    const text = await this.overviewText(group.name, row)
+    const rules = await RuleSets.current()
+    const ix = new RangeIndexer({
+      group: group.name,
+      start: row.first_article,
+      end: row.last_article,
+      rules: { ...rules, samplePercent: 0 },
+      idleChunks: 0,
+      edgeArticles: 0,
+    })
+    ix.addText(text, row.last_article + 1)
+    ix.finish()
+    const batch = ix.take()
+    const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
+    if (!found)
+      throw new Error(
+        `its articles ${row.first_article}–${row.last_article} of ${group.name} no longer show it`
+      )
+    // The catalog row under the pending key (the identity, the group, the first article the edge saw).
+    const chunk: ReleaseChunk = {
+      ...found,
+      name: row.named,
+      source: 'weak-stem',
+      readable: true,
+      finished: true,
+      edge: false,
+      firstArticle: row.first_article,
+      fileTotal: row.file_total ?? found.fileTotal,
+      tool: undefined,
+    }
+    const candidates = batch.candidates
+      .filter((c) => c.releaseKey === found.key)
+      .map((c) => ({ ...c, upload: found.upload }))
+    await db.transaction(async (trx) => {
+      const blockId = await BatchStore.blockOf(trx, range)
+      await BatchStore.writeReleases(
+        trx,
+        { groupId: row.group_id, blockId },
+        row.first_article,
+        [chunk],
+        candidates
+      )
+      await trx.rawQuery(
+        `UPDATE releases SET inner_name = ?, inspected_at = now() WHERE release_key = ? AND inner_name IS NULL`,
+        [row.named.slice(0, 255), row.release_key]
+      )
+      await trx.rawQuery(
+        `UPDATE pending_releases SET status = ?, probed_at = now() WHERE release_key = ?`,
+        [PENDING.named, row.release_key]
+      )
+    })
+  }
+
+  /** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
+  private async overviewText(groupName: string, row: NamedRow): Promise<string> {
+    if (this.corpusDir) {
+      const stored = await db
+        .from('ranges')
+        .where('group_id', row.group_id)
+        .whereNotNull('stored_at')
+        .where('start_id', '<=', row.last_article)
+        .where('end_id', '>=', row.first_article)
+        .orderBy('start_id')
+      const parts: string[] = []
+      let covered = true
+      for (const r of stored) {
+        const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
+        const size = coordinatorConfig.chunkArticles
+        for (
+          let from =
+            range.start +
+            Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
+          from <= Math.min(row.last_article, range.end);
+          from += size
+        ) {
+          const raw = await Corpus.read(
+            range,
+            { from, to: Math.min(from + size - 1, range.end) },
+            this.corpusDir
+          )
+          if (!raw) {
+            covered = false
+            break
+          }
+          parts.push((await inflate(raw)).toString('latin1'))
+        }
+        if (!covered) break
+      }
+      if (
+        covered &&
+        parts.length > 0 &&
+        stored.length > 0 &&
+        Number(stored[0].start_id) <= row.first_article &&
+        Number(stored[stored.length - 1].end_id) >= row.last_article
+      ) {
+        return parts.join('')
+      }
+    }
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: true,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    this.usedAt = Date.now()
+    try {
+      await this.client.group(groupName)
+      return (await this.client.xoverRaw(row.first_article, row.last_article)).toString('latin1')
+    } catch (error) {
+      this.client.close()
+      this.client = null
+      throw error
+    }
+  }
+}

app/services/prober.ts

@@ -9,12 +9,16 @@ import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/i
 import { NntpShare } from '#services/nntp_shares'
 import { DiskSpace } from '#services/disk_space'
 import { decidePar2Completeness } from '#services/par2_completeness'
+import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
+import { PendingNamer } from '#services/pending_namer'
 
 interface Candidate {
   id: number
   releaseId: number
   kind: string
   messageId: string
+  /** A pending release's target (the pending ladder), not a catalog candidate: id and releaseId are 0. */
+  pending?: PendingProbe
 }
 
 type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
@@ -34,6 +38,8 @@ const TIERS: { kind: string; unreadable: boolean }[] = [
   { kind: '7z-tail', unreadable: false },
 ]
 const FLUSH_MS = 2000
+/** Between sweeps that discard the waiting releases of dropped signatures. */
+const SWEEP_MS = 5 * 60_000
 
 /**
  * JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
@@ -72,13 +78,19 @@ export class Prober {
 
   /** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
   private static freshStats() {
-    return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
+    return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, pending: 0, discarded: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
   }
+  /** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
+  private namer = new PendingNamer()
+  private sweptAt = 0
 
   async run() {
     // A previous run's claims: one prober at a time.
     const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
     if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
+    const released = await PendingLadder.release()
+    if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
+    const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
     logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
     const flusher = (async () => {
       while (!this.stopped || this.outcomes.length > 0) {
@@ -108,6 +120,7 @@ export class Prober {
       for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !this.stopped; waited += 1000) await sleep(1000)
     }
     await Promise.all(this.lanes)
+    await namer
     await this.share.leave().catch(() => {})
     // What was claimed and not probed goes back.
     for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
@@ -227,6 +240,37 @@ export class Prober {
   private async claimTiers() {
     // The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again.
     if (await this.space.short()) return
+    await this.claimCatalog()
+    await this.claimPending()
+  }
+
+  /** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
+  private async claimPending() {
+    if (Date.now() - this.sweptAt > SWEEP_MS) {
+      this.sweptAt = Date.now()
+      this.stats.discarded += await PendingLadder.discard()
+    }
+    const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
+    const probes = await PendingLadder.claim(want)
+    if (probes.length === 0) return
+    const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
+      probes.map((p) => p.messageId),
+      PROBE_VERSION,
+    ])
+    const known = new Map(cached.map((c: any) => [c.message_id, c]))
+    for (const probe of probes) {
+      const candidate: Candidate = { id: 0, releaseId: 0, kind: probe.kind, messageId: probe.messageId, pending: probe }
+      const hit: any = known.get(probe.messageId)
+      if (hit) {
+        this.stats.cached++
+        this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
+      } else {
+        this.queue.push(candidate)
+      }
+    }
+  }
+
+  private async claimCatalog() {
     for (const { kind, unreadable } of TIERS) {
       const want = this.batch - this.queue.length
       if (want <= 0) return
@@ -277,18 +321,25 @@ export class Prober {
 
   /** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
   private async flush() {
-    const outcomes = this.outcomes.splice(0)
-    if (outcomes.length === 0) return
-    for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
+    const taken = this.outcomes.splice(0)
+    if (taken.length === 0) return
+    for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
+    // The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
+    const pending = taken.filter((o) => o.candidate.pending)
+    const outcomes = taken.filter((o) => !o.candidate.pending)
+    this.stats.pending += pending.filter((o) => o.status !== 'waiting').length
     await db.transaction(async (trx) => {
-      await trx.rawQuery(
-        `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
-        [jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
-      )
+      if (outcomes.length > 0) {
+        await trx.rawQuery(
+          `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
+          [jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
+        )
+      }
+      await PendingLadder.record(pending.map((o) => ({ probe: o.candidate.pending!, status: o.status, inspection: o.inspection })), trx)
       // Final answers read from the news server, kept by message-id for the next catalog (a reset).
       // One per message-id: the same article is a candidate of more than one release, and ON CONFLICT can't take a
       // row twice in one statement.
-      const fresh = [...new Map(outcomes.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
+      const fresh = [...new Map(taken.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
       if (fresh.length > 0) {
         await trx.rawQuery(
           `INSERT INTO probe_cache (message_id, status, result, version)
@@ -365,7 +416,8 @@ export class Prober {
     const { articles, bytes, done, missing, error } = this.stats
     logger.info(
       `Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
-        `${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache. ` +
+        `${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache; ` +
+        `the pending ladder ${this.stats.pending} probed, ${this.stats.discarded} discarded by signature. ` +
         `Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
         `idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
         `claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`