Prober: the sweep by a signature index, pending rows locked in one order, deadlocked flushes kept

The discard sweep of a dropped signature read the whole pending_releases table per signature (a parallel seq
scan over the poster test, four minutes on the HDD that starved the lanes' claims): a partial index on
(group_id, tool, poster) WHERE status = 0 makes it a range, and the sweep queries a poster's rows by it and a
tool's and group's by the ladder index.

The batch store's pending upsert, the record, and the sweep deadlocked on pending_releases and pending_classes
rows locked in different orders, and a deadlocked flush dropped its outcomes: lockOrder() sorts every
multi-row statement's set the same way, and a flush that deadlocks anyway keeps its outcomes for the next.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/services/pending_ladder.ts

@@ -72,6 +72,19 @@ interface ClassRow {
 const CLASSES_MS = 60_000
 /** A class found short of a claim's share is left alone this long. */
 const EMPTY_MS = 5 * 60_000
+/**
+ * Rows in one order for a statement that locks many of them (pending_releases by key, pending_classes and
+ * pending_signatures by theirs): the batch store, the record, the claim and the sweep all lock the same way, so two
+ * of them can't deadlock on each other (LOG.md (134)).
+ */
+export function lockOrder<T>(rows: Iterable<T>, key: (row: T) => string): T[] {
+  return [...rows].sort((a, b) => {
+    const ka = key(a)
+    const kb = key(b)
+    return ka < kb ? -1 : ka > kb ? 1 : 0
+  })
+}
+
 /** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
 const PRIOR_RATE = 0.15
 const PRIOR_WEIGHT = 10
@@ -282,7 +295,14 @@ class PendingLadder {
          probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
        FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
        WHERE p.release_key = decode(v.release_key, 'hex')`,
-      [JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
+      [
+        JSON.stringify(
+          lockOrder(
+            rows.map(({ final: _final, ...row }) => row),
+            (r) => r.release_key
+          )
+        ),
+      ]
     )
     // The signatures: the poster's own and the tool's and group's together.
     const counts = new Map<
@@ -312,7 +332,7 @@ class PendingLadder {
       `INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
        SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
        ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
-      [JSON.stringify([...counts.values()])]
+      [JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`))]
     )
     await trx.rawQuery(
       `UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float),
@@ -323,7 +343,9 @@ class PendingLadder {
         pendingMinProbes,
         pendingMaxNamed,
         JSON.stringify(
-          [...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
+          lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`).map(
+            ({ tool, group_id, poster }) => ({ tool, group_id, poster })
+          )
         ),
       ]
     )
@@ -360,7 +382,7 @@ class PendingLadder {
        SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
        ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
          waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
-      [JSON.stringify([...classes.values()])]
+      [JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
     )
     await trx.rawQuery(
       `UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float),
@@ -371,7 +393,9 @@ class PendingLadder {
         pendingMinProbes,
         pendingMaxNamed,
         JSON.stringify(
-          [...classes.values()].map(({ tool, group_id, klass }) => ({ tool, group_id, klass }))
+          lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`).map(
+            ({ tool, group_id, klass }) => ({ tool, group_id, klass })
+          )
         ),
       ]
     )
@@ -380,10 +404,10 @@ class PendingLadder {
   /**
    * Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
    * many. Each dropped class or signature is swept once (swept_at; null again when the record drops it anew),
-   * `perSweep` of them at most, each by its index (the claim's for a class, the ladder's for a signature, whose
-   * poster is tested on the rows) in rounds of `limit`. A sweep of a signature's rows takes minutes on the HDD,
-   * so the prober runs this beside the lanes, not in the claim (LOG.md (133)); the edge drops new releases of
-   * dropped signatures and classes itself, so one sweep is all a drop needs.
+   * `perSweep` of them at most, each by its index (the claim's for a class, the signature index for a poster's
+   * rows, the ladder's for a tool's and group's) in rounds of `limit`. The prober runs this beside the lanes, not
+   * in the claim (LOG.md (133)); the edge drops new releases of dropped signatures and classes itself, so one
+   * sweep is all a drop needs.
    */
   async discard(limit = 10_000, perSweep = 20): Promise<number> {
     const { samplePercent } = await RuleSets.current()
@@ -414,11 +438,20 @@ class PendingLadder {
     for (const s of signatures) {
       let rows: any[]
       do {
-        ;({ rows } = await db.rawQuery(
-          `SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
-           WHERE status = ? AND group_id = ? AND tool = ? AND (?::text = '' OR coalesce(poster, '') = ?::text) LIMIT ?`,
-          [PENDING.waiting, s.group_id, s.tool, s.poster, s.poster, limit]
-        ))
+        // The poster's rows by the signature index, a tool's and group's by the ladder's: never a test on the rows,
+        // which was a scan of the whole table per signature on the HDD (LOG.md (134)).
+        ;({ rows } =
+          s.poster === ''
+            ? await db.rawQuery(
+                `SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
+                 WHERE status = ? AND group_id = ? AND tool = ? LIMIT ?`,
+                [PENDING.waiting, s.group_id, s.tool, limit]
+              )
+            : await db.rawQuery(
+                `SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
+                 WHERE status = ? AND group_id = ? AND tool = ? AND poster = ? LIMIT ?`,
+                [PENDING.waiting, s.group_id, s.tool, s.poster, limit]
+              ))
         discarded += await this.discardKeys(rows, samplePercent)
       } while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
       await db.rawQuery(
@@ -434,6 +467,7 @@ class PendingLadder {
     const keys = rows
       .map((r: any) => r.release_key as string)
       .filter((key: string) => !explored(key, samplePercent))
+      .sort()
     if (keys.length === 0) return 0
     const { rows: gone } = await db.rawQuery(
       `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
@@ -457,7 +491,7 @@ class PendingLadder {
         `UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
          FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
          WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
-        [JSON.stringify([...counts.values()])]
+        [JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
       )
     }
     return gone.length