pending:classes without table scans; an index for the prober's startup reset

The first run scanned the 58M-row pending table on the HDD once per
6-hour window and starved the prober and the batch store. Now one scan
classes the finished rows and seeds the record, and the waiting rows go
by windows of created_at through their index. The prober's start puts a
previous run's claims back with a status scan of the same table, 20+
minutes after every deploy: a partial index on the taken rows (migration
6900) makes it instant.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

commands/pending_classes.ts

@@ -6,9 +6,10 @@ import { describeClass } from '#lib/release/index'
 
 /**
  * The pending ladder's classes (LOG.md (130)) on the rows from before them: `klass` (the SQL pending_class) and
- * `target_bytes` (the first target's bytes, when its candidates carry them) set in windows of created_at; the
- * finished rows among them (named, exhausted, naming) seed each class's record, which gets its verdict (dropped) as
- * the ladder gives it; then pending_classes' `waiting` counted afresh from the rows. --report shows the record.
+ * `target_bytes` (the first target's bytes, when its candidates carry them). The finished rows (named, exhausted,
+ * naming) in one scan, seeding each class's record, which gets its verdict (dropped) as the ladder gives it; the
+ * waiting rows in windows of created_at by their index (a scan per window starved the prober on the HDD); then
+ * pending_classes' `waiting` counted afresh from the claim's index. --report shows the record.
  */
 export default class PendingClasses extends BaseCommand {
   static commandName = 'pending:classes'
@@ -25,45 +26,52 @@ export default class PendingClasses extends BaseCommand {
   async run() {
     if (this.report) return this.showReport()
     const hours = this.hours ?? 6
-    const { rows: span } = await db.rawQuery(
-      `SELECT min(created_at) AS lo, max(created_at) AS hi FROM pending_releases WHERE klass = 0`
-    )
-    if (!span[0].lo) {
-      this.logger.info('No rows without a class.')
-    } else {
-      let from = new Date(span[0].lo)
-      const hi = new Date(span[0].hi)
-      let classed = 0
     const started = Date.now()
-      while (from <= hi) {
-        const to = new Date(from.getTime() + hours * 3600_000)
-        // Rows of the window classed; the finished ones among them (named, exhausted, naming) seed the record.
-        const { rows } = await db.rawQuery(
+    // The finished rows (named, exhausted, naming; 6.5M on 2026-11-02) in one pass: a scan of the table, no index
+    // reaches them by status; they seed each class's record.
+    const { rows: seeded } = await db.rawQuery(
       `WITH classed AS (
          UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
            target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
-             WHERE klass = 0 AND created_at >= ? AND created_at < ?
+         WHERE klass = 0 AND status IN (1, 3, 4)
          RETURNING tool, group_id, klass, status),
-           seeded AS (
+       upserted AS (
          INSERT INTO pending_classes (tool, group_id, klass, probed, named)
-             SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed WHERE status IN (1, 3, 4) GROUP BY 1, 2, 3
+         SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed GROUP BY 1, 2, 3
          ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named
          RETURNING 1)
-           SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM seeded) AS seeded`,
-          [from.toISOString(), to.toISOString()]
+       SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM upserted) AS classes`
     )
-        classed += Number(rows[0].classed)
-        from = to
     this.logger.info(
-          `Up to ${to.toISOString().slice(0, 16)}: ${classed} rows classed (${Math.round((Date.now() - started) / 1000)} s).`
+      `Finished rows: ${seeded[0].classed} classed into ${seeded[0].classes} classes (${Math.round((Date.now() - started) / 1000)} s).`
     )
-      }
-      // The record's verdicts on the seeded classes.
     const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
     await db.rawQuery(
       `UPDATE pending_classes SET dropped = true WHERE NOT dropped AND probed >= ?::int AND named::float <= probed * ?::float`,
       [pendingMinProbes, pendingMaxNamed]
     )
+    // The waiting rows by windows of created_at (pending_releases_waiting_idx reaches them).
+    const { rows: span } = await db.rawQuery(
+      `SELECT (SELECT min(created_at) FROM pending_releases WHERE status = 0) AS lo, (SELECT max(created_at) FROM pending_releases WHERE status = 0) AS hi`
+    )
+    if (span[0].lo) {
+      let from = new Date(span[0].lo)
+      const hi = new Date(span[0].hi)
+      let classed = 0
+      while (from <= hi) {
+        const to = new Date(from.getTime() + hours * 3600_000)
+        const result = await db.rawQuery(
+          `UPDATE pending_releases SET klass = pending_class(name, bytes, partial),
+             target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL)
+           WHERE status = 0 AND created_at >= ? AND created_at < ? AND klass = 0`,
+          [from.toISOString(), to.toISOString()]
+        )
+        classed += result.rowCount ?? 0
+        from = to
+        this.logger.info(
+          `Waiting rows up to ${to.toISOString().slice(0, 16)}: ${classed} classed (${Math.round((Date.now() - started) / 1000)} s).`
+        )
+      }
     }
     // The waiting counts from the rows themselves (the claim, the discard and the batch store keep them from here).
     await db.rawQuery(`

database/migrations/1770000006900_pending_taken_index.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The prober's start puts a previous run's claims back (PendingLadder.release: status 9 → 0), which scanned the
+ * 58M-row pending table on the HDD for 20+ minutes after every deploy (LOG.md (130)): a partial index on the few
+ * taken rows makes it instant.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_taken_idx ON pending_releases (release_key) WHERE status = 9`
+    )
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_taken_idx`)
+  }
+}