From 4d46db291e9e44dc87ec4278ce46a972651a411d Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 2 Nov 2026 13:33:48 +0100 Subject: [PATCH] pending:classes without table scans; an index for the prober's startup reset The first run scanned the 58M-row pending table on the HDD once per 6-hour window and starved the prober and the batch store. Now one scan classes the finished rows and seeds the record, and the waiting rows go by windows of created_at through their index. The prober's start puts a previous run's claims back with a status scan of the same table, 20+ minutes after every deploy: a partial index on the taken rows (migration 6900) makes it instant. Co-Authored-By: Claude Fable 5.1 --- commands/pending_classes.ts | 68 +++++++++++-------- .../1770000006900_pending_taken_index.ts | 20 ++++++ 2 files changed, 58 insertions(+), 30 deletions(-) create mode 100644 database/migrations/1770000006900_pending_taken_index.ts diff --git a/commands/pending_classes.ts b/commands/pending_classes.ts index 85a857b..1cd766f 100644 --- a/commands/pending_classes.ts +++ b/commands/pending_classes.ts @@ -6,9 +6,10 @@ import { describeClass } from '#lib/release/index' /** * The pending ladder's classes (LOG.md (130)) on the rows from before them: `klass` (the SQL pending_class) and - * `target_bytes` (the first target's bytes, when its candidates carry them) set in windows of created_at; the - * finished rows among them (named, exhausted, naming) seed each class's record, which gets its verdict (dropped) as - * the ladder gives it; then pending_classes' `waiting` counted afresh from the rows. --report shows the record. + * `target_bytes` (the first target's bytes, when its candidates carry them). The finished rows (named, exhausted, + * naming) in one scan, seeding each class's record, which gets its verdict (dropped) as the ladder gives it; the + * waiting rows in windows of created_at by their index (a scan per window starved the prober on the HDD); then + * pending_classes' `waiting` counted afresh from the claim's index. --report shows the record. */ export default class PendingClasses extends BaseCommand { static commandName = 'pending:classes' @@ -25,45 +26,52 @@ export default class PendingClasses extends BaseCommand { async run() { if (this.report) return this.showReport() const hours = this.hours ?? 6 - const { rows: span } = await db.rawQuery( - `SELECT min(created_at) AS lo, max(created_at) AS hi FROM pending_releases WHERE klass = 0` + const started = Date.now() + // The finished rows (named, exhausted, naming; 6.5M on 2026-11-02) in one pass: a scan of the table, no index + // reaches them by status; they seed each class's record. + const { rows: seeded } = await db.rawQuery( + `WITH classed AS ( + UPDATE pending_releases SET klass = pending_class(name, bytes, partial), + target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL) + WHERE klass = 0 AND status IN (1, 3, 4) + RETURNING tool, group_id, klass, status), + upserted AS ( + INSERT INTO pending_classes (tool, group_id, klass, probed, named) + SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed GROUP BY 1, 2, 3 + ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named + RETURNING 1) + SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM upserted) AS classes` ) - if (!span[0].lo) { - this.logger.info('No rows without a class.') - } else { + this.logger.info( + `Finished rows: ${seeded[0].classed} classed into ${seeded[0].classes} classes (${Math.round((Date.now() - started) / 1000)} s).` + ) + const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe + await db.rawQuery( + `UPDATE pending_classes SET dropped = true WHERE NOT dropped AND probed >= ?::int AND named::float <= probed * ?::float`, + [pendingMinProbes, pendingMaxNamed] + ) + // The waiting rows by windows of created_at (pending_releases_waiting_idx reaches them). + const { rows: span } = await db.rawQuery( + `SELECT (SELECT min(created_at) FROM pending_releases WHERE status = 0) AS lo, (SELECT max(created_at) FROM pending_releases WHERE status = 0) AS hi` + ) + if (span[0].lo) { let from = new Date(span[0].lo) const hi = new Date(span[0].hi) let classed = 0 - const started = Date.now() while (from <= hi) { const to = new Date(from.getTime() + hours * 3600_000) - // Rows of the window classed; the finished ones among them (named, exhausted, naming) seed the record. - const { rows } = await db.rawQuery( - `WITH classed AS ( - UPDATE pending_releases SET klass = pending_class(name, bytes, partial), - target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL) - WHERE klass = 0 AND created_at >= ? AND created_at < ? - RETURNING tool, group_id, klass, status), - seeded AS ( - INSERT INTO pending_classes (tool, group_id, klass, probed, named) - SELECT tool, group_id, klass, count(*), count(*) FILTER (WHERE status IN (1, 4)) FROM classed WHERE status IN (1, 3, 4) GROUP BY 1, 2, 3 - ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named - RETURNING 1) - SELECT (SELECT count(*) FROM classed) AS classed, (SELECT count(*) FROM seeded) AS seeded`, + const result = await db.rawQuery( + `UPDATE pending_releases SET klass = pending_class(name, bytes, partial), + target_bytes = (SELECT min((c->>'bytes')::bigint) FROM jsonb_array_elements(candidates) c WHERE c->>'bytes' IS NOT NULL) + WHERE status = 0 AND created_at >= ? AND created_at < ? AND klass = 0`, [from.toISOString(), to.toISOString()] ) - classed += Number(rows[0].classed) + classed += result.rowCount ?? 0 from = to this.logger.info( - `Up to ${to.toISOString().slice(0, 16)}: ${classed} rows classed (${Math.round((Date.now() - started) / 1000)} s).` + `Waiting rows up to ${to.toISOString().slice(0, 16)}: ${classed} classed (${Math.round((Date.now() - started) / 1000)} s).` ) } - // The record's verdicts on the seeded classes. - const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe - await db.rawQuery( - `UPDATE pending_classes SET dropped = true WHERE NOT dropped AND probed >= ?::int AND named::float <= probed * ?::float`, - [pendingMinProbes, pendingMaxNamed] - ) } // The waiting counts from the rows themselves (the claim, the discard and the batch store keep them from here). await db.rawQuery(` diff --git a/database/migrations/1770000006900_pending_taken_index.ts b/database/migrations/1770000006900_pending_taken_index.ts new file mode 100644 index 0000000..42381a0 --- /dev/null +++ b/database/migrations/1770000006900_pending_taken_index.ts @@ -0,0 +1,20 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * The prober's start puts a previous run's claims back (PendingLadder.release: status 9 → 0), which scanned the + * 58M-row pending table on the HDD for 20+ minutes after every deploy (LOG.md (130)): a partial index on the few + * taken rows makes it instant. + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + this.schema.raw( + `CREATE INDEX CONCURRENTLY IF NOT EXISTS pending_releases_taken_idx ON pending_releases (release_key) WHERE status = 9` + ) + } + + async down() { + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS pending_releases_taken_idx`) + } +}