diff --git a/app/services/prober.ts b/app/services/prober.ts index d097b44..a07269d 100644 --- a/app/services/prober.ts +++ b/app/services/prober.ts @@ -167,8 +167,17 @@ export class Prober { if (want <= 0) return // The starting points chosen here: in the query Postgres drew random() once for all slices. // Over the claimable ones of this tier only: the rest of the queue isn't settled yet. + // Per value of unreadable, each an end of the index: over `unreadable >= false` Postgres walked every waiting + // entry of the kind, 2.5 minutes a claim at 28M archive candidates (LOG.md 2026-10-18 (74)). + const values = unreadable ? [true] : [false, true] + const ends = values + .map(() => `SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable = ?`) + .join(' UNION ALL ') const [{ lo, hi }] = ( - await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable >= ?`, [kind, unreadable]) + await db.rawQuery( + `SELECT min(lo) AS lo, max(hi) AS hi FROM (${ends}) e`, + values.flatMap((v) => [kind, v]) + ) ).rows if (lo === null) continue const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1))) diff --git a/tests/functional/prober.spec.ts b/tests/functional/prober.spec.ts new file mode 100644 index 0000000..70b27b7 --- /dev/null +++ b/tests/functional/prober.spec.ts @@ -0,0 +1,26 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import { Prober } from '#services/prober' + +test.group('Prober', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('a claim takes each tier: the readable tiers also take unreadable candidates, a kind with only one of them too', async ({ assert }) => { + const [row] = await db + .table('releases') + .insert({ release_key: 'pb'.padEnd(40, '0'), identity_key: 'pb'.padEnd(40, '0'), name: 'pb', source: 'subject', groups: '[]', files: 3 }) + .returning('id') + const candidate = (kind: string, unreadable: boolean, ready = true) => ({ + release_id: row.id, kind, message_id: `<${kind}-${unreadable}-${ready}@x>`, file: kind, ready, unreadable, + }) + // One per kind, so each tier's id range is a single id (the slices' starts aren't random then). + await db.table('probe_candidates').multiInsert([candidate('lone', true), candidate('par2', true), candidate('archive', false), candidate('7z-tail', false, false)]) + + const prober = new Prober() as any + await prober.claimTiers() + assert.sameMembers(prober.queue.map((c: { kind: string }) => c.kind), ['lone', 'par2', 'archive']) + const status = Object.fromEntries((await db.from('probe_candidates').select('kind', 'status')).map((c) => [c.kind, c.status])) + assert.deepEqual(status, { 'lone': 'taken', 'par2': 'taken', 'archive': 'taken', '7z-tail': 'waiting' }) + }) +})