Batch store: one lock order for the pending upsert; probe_candidates on the NVMe

writePending upserted pending_releases in chunks of 1,000 rows, each chunk
followed by its class upsert, so one transaction locked releases, classes,
releases, classes, and a later chunk's keys and classes could sort before an
earlier chunk's: the opposite of the ladder's record(), which holds all its
rows and then takes all its classes. Four deadlocks between the two class
upserts in six hours (LOG.md (136)). Now all rows are sorted once and the
classes upserted once, after the last chunk.

Migration 7300 moves probe_candidates and its indexes to the default
tablespace (nvme on homelab): the catalog claim's UPDATE from random points
took 2-3 minutes on the HDD beside the sweep and the autovacuum, with all
lanes waiting. Homelab moved it by hand (83 s, ~8 GB, services stopped); the
migration is a no-op there, and without a default tablespace.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/services/batch_store.ts

@@ -190,9 +190,11 @@ class BatchStore {
       list.push({ kind: c.kind, messageId: c.messageId, file: c.file, bytes: c.bytes })
       targets.set(key, list)
     }
-    let written = 0
-    for (let i = 0; i < releases.length; i += 1000) {
-      const rows = releases.slice(i, i + 1000).map((r) => {
+    // In the record's lock order (LOG.md (136)): all rows sorted, then all classes sorted, in one transaction. The
+    // upsert locks a finished row too, before its WHERE passes it by; a chunk's classes before the next chunk's rows
+    // crossed the ladder's record(), which holds its rows and then takes its classes.
+    const rows = lockOrder(
+      releases.map((r) => {
         const probes = targets.get(slot(r.key, r.upload)) ?? []
         return {
           release_key: releaseKey(r.key, groupId, r.firstArticle).toString('hex'),
@@ -214,7 +216,12 @@ class BatchStore {
           // The first target's article (by effort): the cheapest probe first within a class (LOG.md (130)).
           target_bytes: byEffort(probes)[0]?.bytes ?? null,
         }
-      })
+      }),
+      (r) => r.release_key
+    )
+    let written = 0
+    const classes = new Map<string, { tool: string; klass: number; waiting: number }>()
+    for (let i = 0; i < rows.length; i += 1000) {
       const { rows: done } = await trx.rawQuery(
         `INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
                                        files, segments, bytes, partial, finished, file_total, tool, candidates, klass, target_bytes)
@@ -232,29 +239,27 @@ class BatchStore {
            target_bytes = coalesce(pending_releases.target_bytes, excluded.target_bytes)
          WHERE NOT pending_releases.finished
          RETURNING release_key, tool, klass, (xmax = 0) AS inserted`,
-        // In the record's lock order: the upsert locks a finished row too, before its WHERE passes it by.
-        [groupId, JSON.stringify(lockOrder(rows, (r) => r.release_key))]
+        [groupId, JSON.stringify(rows.slice(i, i + 1000))]
       )
       written += done.length
       // Each class seen gets its row in the ladder's record, so the claim explores it (pending_classes).
-      const classes = new Map<string, { tool: string; klass: number; waiting: number }>()
       for (const d of done) {
         const key = `${d.tool}\t${d.klass}`
         const entry = classes.get(key) ?? { tool: d.tool, klass: Number(d.klass), waiting: 0 }
         if (d.inserted) entry.waiting++
         classes.set(key, entry)
       }
-      if (classes.size > 0) {
-        await trx.rawQuery(
-          `INSERT INTO pending_classes (tool, group_id, klass, waiting)
-           SELECT v.tool, ?, v.klass, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, klass int, waiting int)
-           ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = pending_classes.waiting + excluded.waiting`,
-          [
-            groupId,
-            JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${groupId}\t${c.klass}`)),
-          ]
-        )
-      }
+    }
+    if (classes.size > 0) {
+      await trx.rawQuery(
+        `INSERT INTO pending_classes (tool, group_id, klass, waiting)
+         SELECT v.tool, ?, v.klass, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, klass int, waiting int)
+         ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = pending_classes.waiting + excluded.waiting`,
+        [
+          groupId,
+          JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${groupId}\t${c.klass}`)),
+        ]
+      )
     }
     return written
   }