usenet-indexer/app/services/pending_ladder.ts

import db from '@adonisjs/lucid/services/db'
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
import coordinatorConfig from '#config/coordinator'
import { explored } from '#lib/rules/index'
import { usableName, type Inspection } from '#lib/probe/index'
import RuleSets from '#services/rule_sets'
import { twinKey, twinNames } from '#services/hash_twins'

/** pending_releases.status (migration 6400). */
export const PENDING = {
  waiting: 0,
  named: 1,
  discarded: 2,
  exhausted: 3,
  naming: 4,
  taken: 9,
} as const

/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const

export interface PendingTarget {
  kind: string
  messageId: string
  file: string
  /** The target article's bytes (candidates from 2026-11-02 on). */
  bytes?: number
}

/** One probe the ladder hands the prober: the next target of a pending release. */
export interface PendingProbe {
  /** pending_releases.release_key as hex. */
  releaseKey: string
  kind: string
  messageId: string
  file: string
  tool: string
  groupId: number
  /** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
  klass: number
  poster: string
  /** Targets probed before this one. */
  tries: number
  targets: number
}

export type PendingOutcome = {
  probe: PendingProbe
  status: 'done' | 'missing' | 'error' | 'waiting'
  inspection?: Inspection
}

export function byEffort(targets: PendingTarget[]): PendingTarget[] {
  const rank = (kind: string) => {
    const i = (EFFORT as readonly string[]).indexOf(kind)
    return i < 0 ? EFFORT.length : i
  }
  return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
}

/** A class's record (pending_classes), for the claim's order. */
interface ClassRow {
  tool: string
  groupId: number
  klass: number
  probed: number
  named: number
  waiting: number
}

/** The record is read again this often. */
const CLASSES_MS = 60_000
/** A class found short of a claim's share is left alone this long. */
const EMPTY_MS = 5 * 60_000
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
const PRIOR_RATE = 0.15
const PRIOR_WEIGHT = 10
/** One class takes at most this share of a claim: the rest explores the classes after it. */
const CLASS_SHARE = 0.25

/** The expected naming rate of a class: its record shrunk towards the prior. */
export function classScore(c: { probed: number; named: number }): number {
  return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
}

/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
function claimed(rows: any[]): any[] {
  const bytes = (r: any) =>
    r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
  return [...rows].sort(
    (a, b) =>
      bytes(a) - bytes(b) ||
      new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
      Number(a.first_article) - Number(b.first_article)
  )
}

function toProbe(r: any): PendingProbe {
  const targets = byEffort(r.candidates as PendingTarget[])
  const tries = Number(r.tries)
  const target = targets[Math.min(tries, targets.length - 1)]
  return {
    releaseKey: r.release_key,
    kind: target.kind,
    messageId: target.messageId,
    file: target.file,
    tool: r.tool,
    groupId: Number(r.group_id),
    klass: Number(r.klass ?? 0),
    poster: r.poster,
    tries,
    targets: targets.length,
  }
}

/**
 * The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
 * handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
 * every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
 * class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
 * or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
 * a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
 * classes with the best record first (classScore), the cheapest target first within one.
 */
class PendingLadder {
  /** A previous prober's claims back in the queue. */
  async release(): Promise<number> {
    const { rowCount } = await db.rawQuery(
      `UPDATE pending_releases SET status = ? WHERE status = ?`,
      [PENDING.waiting, PENDING.taken]
    )
    return rowCount ?? 0
  }

  /** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
  private classes: ClassRow[] = []
  private classesAt = 0
  private empty = new Map<string, number>()

  /** The record read again on the next claim (the tests). */
  forget() {
    this.classesAt = 0
    this.empty.clear()
  }

  /** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
  private async classRows(): Promise<ClassRow[]> {
    if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
    const { rows } = await db.rawQuery(
      `SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
    )
    this.classes = rows
      .map((r: any) => ({
        tool: r.tool as string,
        groupId: Number(r.group_id),
        klass: Number(r.klass),
        probed: Number(r.probed),
        named: Number(r.named),
        waiting: Number(r.waiting),
      }))
      .sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
    this.classesAt = Date.now()
    return this.classes
  }

  /**
   * The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
   * at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
   * and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
   * classes, status 0 of a class without a row).
   */
  async claim(n: number): Promise<PendingProbe[]> {
    if (n <= 0) return []
    const out: PendingProbe[] = []
    const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
    const now = Date.now()
    for (const c of await this.classRows()) {
      if (out.length >= n) break
      const key = `${c.tool}\t${c.groupId}\t${c.klass}`
      if ((this.empty.get(key) ?? 0) > now) continue
      const want = Math.min(perClass, n - out.length)
      const { rows } = await db.rawQuery(
        `UPDATE pending_releases p SET status = ? FROM (
           SELECT release_key FROM pending_releases
           WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
           ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
         WHERE p.release_key = c.release_key
         RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
        [PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
      )
      if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
      if (rows.length > 0) {
        await db.rawQuery(
          `UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
          [rows.length, c.tool, c.groupId, c.klass]
        )
      }
      out.push(...claimed(rows).map(toProbe))
    }
    if (out.length < n) {
      const { rows } = await db.rawQuery(
        `UPDATE pending_releases p SET status = ? FROM (
           SELECT release_key FROM pending_releases
           WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
           ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
         WHERE p.release_key = c.release_key
         RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
        [PENDING.taken, PENDING.waiting, n - out.length]
      )
      out.push(...claimed(rows).map(toProbe))
    }
    return out
  }

  /**
   * What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
   * end of the ladder (status 3); the finished climbs counted on their signatures.
   */
  async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
    if (outcomes.length === 0) return
    // A twin by content names what the probe itself didn't (hash_twins.ts).
    const twins = await twinNames(
      outcomes
        .filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
        .map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
      trx
    )
    const rows = outcomes.map((o) => {
      const p = o.probe
      if (o.status === 'waiting') {
        return {
          release_key: p.releaseKey,
          status: PENDING.waiting,
          tries: p.tries,
          named: null,
          inspection: null,
          final: null,
        }
      }
      const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
      const twin =
        o.status === 'done' && o.inspection?.hash16k
          ? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
          : undefined
      const name = probed ?? twin ?? null
      const tries = p.tries + 1
      const inspection = {
        kind: p.kind,
        status: o.status,
        result: o.inspection ? { ...o.inspection, par2: undefined } : null,
      }
      if (name)
        return {
          release_key: p.releaseKey,
          status: PENDING.naming,
          tries: 0,
          named: name,
          inspection,
          final: 'named',
        }
      if (tries >= p.targets)
        return {
          release_key: p.releaseKey,
          status: PENDING.exhausted,
          tries,
          named: null,
          inspection,
          final: 'exhausted',
        }
      return {
        release_key: p.releaseKey,
        status: PENDING.waiting,
        tries,
        named: null,
        inspection,
        final: null,
      }
    })
    await trx.rawQuery(
      `UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
         inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
         probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
       FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
       WHERE p.release_key = decode(v.release_key, 'hex')`,
      [JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
    )
    // The signatures: the poster's own and the tool's and group's together.
    const counts = new Map<
      string,
      { tool: string; group_id: number; poster: string; probed: number; named: number }
    >()
    for (const [i, o] of outcomes.entries()) {
      const final = rows[i].final
      if (!final) continue
      for (const poster of [o.probe.poster, '']) {
        const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
        const entry = counts.get(key) ?? {
          tool: o.probe.tool,
          group_id: o.probe.groupId,
          poster,
          probed: 0,
          named: 0,
        }
        entry.probed++
        if (final === 'named') entry.named++
        counts.set(key, entry)
      }
    }
    if (counts.size === 0) return
    const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
    await trx.rawQuery(
      `INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
       SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
       ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
      [JSON.stringify([...counts.values()])]
    )
    await trx.rawQuery(
      `UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
       FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
       WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
      [
        pendingMinProbes,
        pendingMaxNamed,
        JSON.stringify(
          [...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
        ),
      ]
    )
    // The classes: the same record per (tool, group, class), and a probe given back is waiting again.
    const classes = new Map<
      string,
      {
        tool: string
        group_id: number
        klass: number
        probed: number
        named: number
        waiting: number
      }
    >()
    for (const [i, o] of outcomes.entries()) {
      const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
      const entry = classes.get(key) ?? {
        tool: o.probe.tool,
        group_id: o.probe.groupId,
        klass: o.probe.klass,
        probed: 0,
        named: 0,
        waiting: 0,
      }
      const final = rows[i].final
      if (final) entry.probed++
      if (final === 'named') entry.named++
      if (o.status === 'waiting') entry.waiting++
      classes.set(key, entry)
    }
    await trx.rawQuery(
      `INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
       SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
       ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
         waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
      [JSON.stringify([...classes.values()])]
    )
    await trx.rawQuery(
      `UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
       FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
       WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
      [
        pendingMinProbes,
        pendingMaxNamed,
        JSON.stringify(
          [...classes.values()].map(({ tool, group_id, klass }) => ({ tool, group_id, klass }))
        ),
      ]
    )
  }

  /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
  async discard(limit = 10_000): Promise<number> {
    const { samplePercent } = await RuleSets.current()
    const { rows } = await db.rawQuery(
      `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
       WHERE p.status = ? AND (EXISTS (
         SELECT 1 FROM pending_signatures s
         WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
         OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass))
       LIMIT ?`,
      [PENDING.waiting, limit]
    )
    const keys = rows
      .map((r: any) => r.release_key as string)
      .filter((key: string) => !explored(key, samplePercent))
    if (keys.length === 0) return 0
    const { rows: gone } = await db.rawQuery(
      `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
       RETURNING tool, group_id, klass`,
      [PENDING.discarded, PENDING.waiting, keys]
    )
    const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
    for (const g of gone) {
      const key = `${g.tool}\t${g.group_id}\t${g.klass}`
      const entry = counts.get(key) ?? {
        tool: g.tool,
        group_id: Number(g.group_id),
        klass: Number(g.klass),
        n: 0,
      }
      entry.n++
      counts.set(key, entry)
    }
    if (counts.size > 0) {
      await db.rawQuery(
        `UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
         FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
         WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
        [JSON.stringify([...counts.values()])]
      )
    }
    return gone.length
  }
}

export default new PendingLadder()