The pending ladder in the prober: unnamed releases probed cheapest first, signatures learned, the named moved into the catalog

pending_ladder.ts claims the oldest waiting pending releases' next target
(par2, archive, lone, 7z-tail in that order) as at least PENDING_SHARE of each
claim; a usable name (the largest named file of a par2 set or a listing, or a
lone file's yEnc name, neither packaging nor random) hands the release to the
namer, no name the next target or the end of the climb; climbs count on
pending_signatures per (tool, group, poster) and (tool, group), and a signature
with 100 climbs and 0.5% named or fewer is dropped: its waiting releases are
discarded without a probe, the rules' sample excepted. pending_namer.ts reads
a named release's articles again from the corpus or by XOVER and writes it to
the catalog under the pending key (source weak-stem, the name as inner name)
with its chunk and candidates. Migration 6400; config PENDING_*.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/services/pending_ladder.ts

@@ -0,0 +1,230 @@
+import db from '@adonisjs/lucid/services/db'
+import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
+import coordinatorConfig from '#config/coordinator'
+import { explored } from '#lib/rules/index'
+import { usableName, type Inspection } from '#lib/probe/index'
+import RuleSets from '#services/rule_sets'
+
+/** pending_releases.status (migration 6400). */
+export const PENDING = {
+  waiting: 0,
+  named: 1,
+  discarded: 2,
+  exhausted: 3,
+  naming: 4,
+  taken: 9,
+} as const
+
+/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
+export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
+
+export interface PendingTarget {
+  kind: string
+  messageId: string
+  file: string
+}
+
+/** One probe the ladder hands the prober: the next target of a pending release. */
+export interface PendingProbe {
+  /** pending_releases.release_key as hex. */
+  releaseKey: string
+  kind: string
+  messageId: string
+  file: string
+  tool: string
+  groupId: number
+  poster: string
+  /** Targets probed before this one. */
+  tries: number
+  targets: number
+}
+
+export type PendingOutcome = {
+  probe: PendingProbe
+  status: 'done' | 'missing' | 'error' | 'waiting'
+  inspection?: Inspection
+}
+
+export function byEffort(targets: PendingTarget[]): PendingTarget[] {
+  const rank = (kind: string) => {
+    const i = (EFFORT as readonly string[]).indexOf(kind)
+    return i < 0 ? EFFORT.length : i
+  }
+  return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
+}
+
+/**
+ * The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
+ * handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
+ * every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
+ * signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
+ * a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
+ */
+class PendingLadder {
+  /** A previous prober's claims back in the queue. */
+  async release(): Promise<number> {
+    const { rowCount } = await db.rawQuery(
+      `UPDATE pending_releases SET status = ? WHERE status = ?`,
+      [PENDING.waiting, PENDING.taken]
+    )
+    return rowCount ?? 0
+  }
+
+  /** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
+  async claim(n: number): Promise<PendingProbe[]> {
+    if (n <= 0) return []
+    const { rows } = await db.rawQuery(
+      `UPDATE pending_releases p SET status = ? FROM (
+         SELECT release_key FROM pending_releases
+         WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
+         ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
+       WHERE p.release_key = c.release_key
+       RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
+      [PENDING.taken, PENDING.waiting, n]
+    )
+    return rows.map((r: any) => {
+      const targets = byEffort(r.candidates as PendingTarget[])
+      const tries = Number(r.tries)
+      const target = targets[Math.min(tries, targets.length - 1)]
+      return {
+        releaseKey: r.release_key,
+        kind: target.kind,
+        messageId: target.messageId,
+        file: target.file,
+        tool: r.tool,
+        groupId: Number(r.group_id),
+        poster: r.poster,
+        tries,
+        targets: targets.length,
+      }
+    })
+  }
+
+  /**
+   * What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
+   * end of the ladder (status 3); the finished climbs counted on their signatures.
+   */
+  async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
+    if (outcomes.length === 0) return
+    const rows = outcomes.map((o) => {
+      const p = o.probe
+      if (o.status === 'waiting') {
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.waiting,
+          tries: p.tries,
+          named: null,
+          inspection: null,
+          final: null,
+        }
+      }
+      const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
+      const tries = p.tries + 1
+      const inspection = {
+        kind: p.kind,
+        status: o.status,
+        result: o.inspection ? { ...o.inspection, par2: undefined } : null,
+      }
+      if (name)
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.naming,
+          tries: 0,
+          named: name,
+          inspection,
+          final: 'named',
+        }
+      if (tries >= p.targets)
+        return {
+          release_key: p.releaseKey,
+          status: PENDING.exhausted,
+          tries,
+          named: null,
+          inspection,
+          final: 'exhausted',
+        }
+      return {
+        release_key: p.releaseKey,
+        status: PENDING.waiting,
+        tries,
+        named: null,
+        inspection,
+        final: null,
+      }
+    })
+    await trx.rawQuery(
+      `UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
+         inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
+         probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
+       FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
+       WHERE p.release_key = decode(v.release_key, 'hex')`,
+      [JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
+    )
+    // The signatures: the poster's own and the tool's and group's together.
+    const counts = new Map<
+      string,
+      { tool: string; group_id: number; poster: string; probed: number; named: number }
+    >()
+    for (const [i, o] of outcomes.entries()) {
+      const final = rows[i].final
+      if (!final) continue
+      for (const poster of [o.probe.poster, '']) {
+        const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
+        const entry = counts.get(key) ?? {
+          tool: o.probe.tool,
+          group_id: o.probe.groupId,
+          poster,
+          probed: 0,
+          named: 0,
+        }
+        entry.probed++
+        if (final === 'named') entry.named++
+        counts.set(key, entry)
+      }
+    }
+    if (counts.size === 0) return
+    const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
+    await trx.rawQuery(
+      `INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
+       SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
+       ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
+      [JSON.stringify([...counts.values()])]
+    )
+    await trx.rawQuery(
+      `UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
+       FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
+       WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
+      [
+        pendingMinProbes,
+        pendingMaxNamed,
+        JSON.stringify(
+          [...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
+        ),
+      ]
+    )
+  }
+
+  /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
+  async discard(limit = 10_000): Promise<number> {
+    const { samplePercent } = await RuleSets.current()
+    const { rows } = await db.rawQuery(
+      `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
+       WHERE p.status = ? AND EXISTS (
+         SELECT 1 FROM pending_signatures s
+         WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
+       LIMIT ?`,
+      [PENDING.waiting, limit]
+    )
+    const keys = rows
+      .map((r: any) => r.release_key as string)
+      .filter((key: string) => !explored(key, samplePercent))
+    if (keys.length === 0) return 0
+    const { rowCount } = await db.rawQuery(
+      `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
+      [PENDING.discarded, PENDING.waiting, keys]
+    )
+    return rowCount ?? 0
+  }
+}
+
+export default new PendingLadder()