import db from '@adonisjs/lucid/services/db' import type { TransactionClientContract } from '@adonisjs/lucid/types/database' import coordinatorConfig from '#config/coordinator' import { explored } from '#lib/rules/index' import { usableName, type Inspection } from '#lib/probe/index' import RuleSets from '#services/rule_sets' /** pending_releases.status (migration 6400). */ export const PENDING = { waiting: 0, named: 1, discarded: 2, exhausted: 3, naming: 4, taken: 9, } as const /** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */ export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const export interface PendingTarget { kind: string messageId: string file: string } /** One probe the ladder hands the prober: the next target of a pending release. */ export interface PendingProbe { /** pending_releases.release_key as hex. */ releaseKey: string kind: string messageId: string file: string tool: string groupId: number poster: string /** Targets probed before this one. */ tries: number targets: number } export type PendingOutcome = { probe: PendingProbe status: 'done' | 'missing' | 'error' | 'waiting' inspection?: Inspection } export function byEffort(targets: PendingTarget[]): PendingTarget[] { const rank = (kind: string) => { const i = (EFFORT as readonly string[]).indexOf(kind) return i < 0 ? EFFORT.length : i } return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind)) } /** * The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name * handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it; * every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a * signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe, * a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. */ class PendingLadder { /** A previous prober's claims back in the queue. */ async release(): Promise { const { rowCount } = await db.rawQuery( `UPDATE pending_releases SET status = ? WHERE status = ?`, [PENDING.waiting, PENDING.taken] ) return rowCount ?? 0 } /** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */ async claim(n: number): Promise { if (n <= 0) return [] const { rows } = await db.rawQuery( `UPDATE pending_releases p SET status = ? FROM ( SELECT release_key FROM pending_releases WHERE status = ? AND finished AND tries < jsonb_array_length(candidates) ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c WHERE p.release_key = c.release_key RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`, [PENDING.taken, PENDING.waiting, n] ) return rows.map((r: any) => { const targets = byEffort(r.candidates as PendingTarget[]) const tries = Number(r.tries) const target = targets[Math.min(tries, targets.length - 1)] return { releaseKey: r.release_key, kind: target.kind, messageId: target.messageId, file: target.file, tool: r.tool, groupId: Number(r.group_id), poster: r.poster, tries, targets: targets.length, } }) } /** * What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the * end of the ladder (status 3); the finished climbs counted on their signatures. */ async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) { if (outcomes.length === 0) return const rows = outcomes.map((o) => { const p = o.probe if (o.status === 'waiting') { return { release_key: p.releaseKey, status: PENDING.waiting, tries: p.tries, named: null, inspection: null, final: null, } } const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null const tries = p.tries + 1 const inspection = { kind: p.kind, status: o.status, result: o.inspection ? { ...o.inspection, par2: undefined } : null, } if (name) return { release_key: p.releaseKey, status: PENDING.naming, tries: 0, named: name, inspection, final: 'named', } if (tries >= p.targets) return { release_key: p.releaseKey, status: PENDING.exhausted, tries, named: null, inspection, final: 'exhausted', } return { release_key: p.releaseKey, status: PENDING.waiting, tries, named: null, inspection, final: null, } }) await trx.rawQuery( `UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named), inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END, probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb) WHERE p.release_key = decode(v.release_key, 'hex')`, [JSON.stringify(rows.map(({ final: _final, ...row }) => row))] ) // The signatures: the poster's own and the tool's and group's together. const counts = new Map< string, { tool: string; group_id: number; poster: string; probed: number; named: number } >() for (const [i, o] of outcomes.entries()) { const final = rows[i].final if (!final) continue for (const poster of [o.probe.poster, '']) { const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}` const entry = counts.get(key) ?? { tool: o.probe.tool, group_id: o.probe.groupId, poster, probed: 0, named: 0, } entry.probed++ if (final === 'named') entry.named++ counts.set(key, entry) } } if (counts.size === 0) return const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe await trx.rawQuery( `INSERT INTO pending_signatures (tool, group_id, poster, probed, named) SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int) ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`, [JSON.stringify([...counts.values()])] ) await trx.rawQuery( `UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float) FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text) WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`, [ pendingMinProbes, pendingMaxNamed, JSON.stringify( [...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster })) ), ] ) } /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */ async discard(limit = 10_000): Promise { const { samplePercent } = await RuleSets.current() const { rows } = await db.rawQuery( `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p WHERE p.status = ? AND EXISTS ( SELECT 1 FROM pending_signatures s WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, ''))) LIMIT ?`, [PENDING.waiting, limit] ) const keys = rows .map((r: any) => r.release_key as string) .filter((key: string) => !explored(key, samplePercent)) if (keys.length === 0) return 0 const { rowCount } = await db.rawQuery( `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`, [PENDING.discarded, PENDING.waiting, keys] ) return rowCount ?? 0 } } export default new PendingLadder()