Prober: claim only candidates of settled releases, without the join

The settler marks a settled release's candidates ready (and unreadable when
it had no readable name). Claims took the candidates by joining releases
from random points of the queue; in its newer half nothing had settled, so
a slice scanned to the end (0.3 s each, 20 slices and 5 tiers a claim) and
the prober stood still: 2.4M waiting, 0 probed a minute. Now an index over
the claimable rows only, the slices drawn within them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/prober.ts

@@ -166,15 +166,19 @@ export class Prober {
       const want = coordinatorConfig.probe.batch - this.queue.length
       if (want <= 0) return
       // The starting points chosen here: in the query Postgres drew random() once for all slices.
-      const [{ lo, hi }] = (await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates`)).rows
+      // Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
+      const [{ lo, hi }] = (
+        await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable >= ?`, [kind, unreadable])
+      ).rows
+      if (lo === null) continue
       const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
       const { rows } = await db.rawQuery(
         `UPDATE probe_candidates SET status = 'taken' WHERE id IN (
            SELECT pc.id FROM unnest(?::bigint[]) AS o(start)
            CROSS JOIN LATERAL (
-             SELECT pc.id FROM probe_candidates pc JOIN releases r ON r.id = pc.release_id
+             SELECT pc.id FROM probe_candidates pc
-             WHERE pc.status = 'waiting' AND pc.kind = ? AND r.settled_at IS NOT NULL AND (NOT ? OR NOT r.readable) AND pc.id >= o.start
+             WHERE pc.status = 'waiting' AND pc.ready AND pc.kind = ? AND pc.unreadable >= ? AND pc.id >= o.start
-             ORDER BY pc.id LIMIT ? FOR UPDATE OF pc SKIP LOCKED) pc)
+             ORDER BY pc.kind, pc.unreadable, pc.id LIMIT ? FOR UPDATE SKIP LOCKED) pc)
          RETURNING id, release_id, kind, message_id`,
         [starts, kind, unreadable, Math.ceil(want / SLICES)]
       )

app/services/settler.ts

@@ -21,14 +21,20 @@ class Settler {
   async run(): Promise<number> {
     let settled = 0
     while (true) {
-      const { rowCount } = await db.rawQuery(
+      // Their probe candidates become claimable (Prober.claim), the unreadable ones first.
-        `UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
+      const { rows } = await db.rawQuery(
-         WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
+        `WITH settled AS (
-           AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)`,
+           UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
+           WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
+             AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
+           RETURNING id, readable),
+         marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id)
+         SELECT count(*)::int AS n FROM settled`,
         [QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
       )
-      settled += rowCount ?? 0
+      const n = rows[0].n
-      if ((rowCount ?? 0) < BATCH) return settled
+      settled += n
+      if (n < BATCH) return settled
     }
   }
 }

database/migrations/1770000002500_probe_candidates_ready.ts

@@ -0,0 +1,30 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Candidates the prober may claim (Prober.claim): `ready` once their release
+ * settled (the settler sets it), `unreadable` when that release had no
+ * readable name then. The claims joined releases for both and started at
+ * random points of the queue; in its newer half nothing had settled, so a
+ * slice scanned to the end (0.3 s a slice, 20 slices, 5 tiers: the prober
+ * stood still for minutes per claim; LOG.md 2026-10-12 (20)). With an index
+ * over the claimable rows only.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`ALTER TABLE probe_candidates ADD COLUMN IF NOT EXISTS ready boolean NOT NULL DEFAULT false`)
+    this.schema.raw(`ALTER TABLE probe_candidates ADD COLUMN IF NOT EXISTS unreadable boolean NOT NULL DEFAULT false`)
+    this.schema.raw(
+      `UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
+       WHERE r.id = pc.release_id AND r.settled_at IS NOT NULL AND NOT pc.ready`
+    )
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_ready_idx ON probe_candidates (kind, unreadable, id) WHERE status = 'waiting' AND ready`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_kind_waiting_idx`)
+  }
+
+  async down() {
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_ready_idx`)
+    this.schema.raw(`ALTER TABLE probe_candidates DROP COLUMN IF EXISTS ready, DROP COLUMN IF EXISTS unreadable`)
+  }
+}

database/schema.ts

@@ -169,7 +169,7 @@ export class ProbeCacheSchema extends BaseModel {
 }
 
 export class ProbeCandidateSchema extends BaseModel {
-  static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'releaseId', 'status'] as const
+  static $columns = ['createdAt', 'file', 'id', 'kind', 'messageId', 'ready', 'releaseId', 'status', 'unreadable'] as const
   $columns = ProbeCandidateSchema.$columns
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
@@ -182,9 +182,13 @@ export class ProbeCandidateSchema extends BaseModel {
   @column()
   declare messageId: string
   @column()
+  declare ready: boolean
+  @column()
   declare releaseId: bigint | number
   @column()
   declare status: string
+  @column()
+  declare unreadable: boolean
 }
 
 export class RangeSchema extends BaseModel {

tests/functional/settler.spec.ts

@@ -24,7 +24,15 @@ test.group('Settler', (group) => {
     })
     const minutesAgo = (m: number) => new Date(Date.now() - m * 60_000)
     await db.table('releases').multiInsert([row('old', minutesAgo(31), posted), row('fresh', minutesAgo(5), posted), row('tail', minutesAgo(31), minutesAgo(40))])
+    const ids = Object.fromEntries((await db.from('releases').select('id', 'name')).map((r) => [r.name, r.id]))
+    await db.table('probe_candidates').multiInsert([
+      { release_id: ids.old, kind: 'lone', message_id: '<a@x>', file: 'a' },
+      { release_id: ids.fresh, kind: 'lone', message_id: '<b@x>', file: 'b' },
+    ])
     assert.equal(await Settler.run(), 1)
+    // The settled release's candidates are claimable now, as unreadable (readable defaults to false).
+    const ready = await db.from('probe_candidates').orderBy('message_id').select('ready', 'unreadable')
+    assert.deepEqual(ready.map((r) => [r.ready, r.unreadable]), [[true, true], [false, false]])
     const settled = (await db.from('releases').whereNotNull('settled_at').select('name')).map((r) => r.name)
     assert.deepEqual(settled, ['old'])
   })