usenet-indexer/app/services/pending_ladder.ts

import db from '@adonisjs/lucid/services/db'
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
import coordinatorConfig from '#config/coordinator'
import { explored } from '#lib/rules/index'
import { usableName, type Inspection } from '#lib/probe/index'
import RuleSets from '#services/rule_sets'

/** pending_releases.status (migration 6400). */
export const PENDING = {
  waiting: 0,
  named: 1,
  discarded: 2,
  exhausted: 3,
  naming: 4,
  taken: 9,
} as const

/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const

export interface PendingTarget {
  kind: string
  messageId: string
  file: string
}

/** One probe the ladder hands the prober: the next target of a pending release. */
export interface PendingProbe {
  /** pending_releases.release_key as hex. */
  releaseKey: string
  kind: string
  messageId: string
  file: string
  tool: string
  groupId: number
  poster: string
  /** Targets probed before this one. */
  tries: number
  targets: number
}

export type PendingOutcome = {
  probe: PendingProbe
  status: 'done' | 'missing' | 'error' | 'waiting'
  inspection?: Inspection
}

export function byEffort(targets: PendingTarget[]): PendingTarget[] {
  const rank = (kind: string) => {
    const i = (EFFORT as readonly string[]).indexOf(kind)
    return i < 0 ? EFFORT.length : i
  }
  return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
}

/**
 * The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
 * handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
 * every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
 * signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
 * a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
 */
class PendingLadder {
  /** A previous prober's claims back in the queue. */
  async release(): Promise<number> {
    const { rowCount } = await db.rawQuery(
      `UPDATE pending_releases SET status = ? WHERE status = ?`,
      [PENDING.waiting, PENDING.taken]
    )
    return rowCount ?? 0
  }

  /** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
  async claim(n: number): Promise<PendingProbe[]> {
    if (n <= 0) return []
    const { rows } = await db.rawQuery(
      `UPDATE pending_releases p SET status = ? FROM (
         SELECT release_key FROM pending_releases
         WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
         ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
       WHERE p.release_key = c.release_key
       RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
      [PENDING.taken, PENDING.waiting, n]
    )
    return rows.map((r: any) => {
      const targets = byEffort(r.candidates as PendingTarget[])
      const tries = Number(r.tries)
      const target = targets[Math.min(tries, targets.length - 1)]
      return {
        releaseKey: r.release_key,
        kind: target.kind,
        messageId: target.messageId,
        file: target.file,
        tool: r.tool,
        groupId: Number(r.group_id),
        poster: r.poster,
        tries,
        targets: targets.length,
      }
    })
  }

  /**
   * What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
   * end of the ladder (status 3); the finished climbs counted on their signatures.
   */
  async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
    if (outcomes.length === 0) return
    const rows = outcomes.map((o) => {
      const p = o.probe
      if (o.status === 'waiting') {
        return {
          release_key: p.releaseKey,
          status: PENDING.waiting,
          tries: p.tries,
          named: null,
          inspection: null,
          final: null,
        }
      }
      const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
      const tries = p.tries + 1
      const inspection = {
        kind: p.kind,
        status: o.status,
        result: o.inspection ? { ...o.inspection, par2: undefined } : null,
      }
      if (name)
        return {
          release_key: p.releaseKey,
          status: PENDING.naming,
          tries: 0,
          named: name,
          inspection,
          final: 'named',
        }
      if (tries >= p.targets)
        return {
          release_key: p.releaseKey,
          status: PENDING.exhausted,
          tries,
          named: null,
          inspection,
          final: 'exhausted',
        }
      return {
        release_key: p.releaseKey,
        status: PENDING.waiting,
        tries,
        named: null,
        inspection,
        final: null,
      }
    })
    await trx.rawQuery(
      `UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
         inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
         probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
       FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
       WHERE p.release_key = decode(v.release_key, 'hex')`,
      [JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
    )
    // The signatures: the poster's own and the tool's and group's together.
    const counts = new Map<
      string,
      { tool: string; group_id: number; poster: string; probed: number; named: number }
    >()
    for (const [i, o] of outcomes.entries()) {
      const final = rows[i].final
      if (!final) continue
      for (const poster of [o.probe.poster, '']) {
        const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
        const entry = counts.get(key) ?? {
          tool: o.probe.tool,
          group_id: o.probe.groupId,
          poster,
          probed: 0,
          named: 0,
        }
        entry.probed++
        if (final === 'named') entry.named++
        counts.set(key, entry)
      }
    }
    if (counts.size === 0) return
    const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
    await trx.rawQuery(
      `INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
       SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
       ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
      [JSON.stringify([...counts.values()])]
    )
    await trx.rawQuery(
      `UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
       FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
       WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
      [
        pendingMinProbes,
        pendingMaxNamed,
        JSON.stringify(
          [...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
        ),
      ]
    )
  }

  /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
  async discard(limit = 10_000): Promise<number> {
    const { samplePercent } = await RuleSets.current()
    const { rows } = await db.rawQuery(
      `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
       WHERE p.status = ? AND EXISTS (
         SELECT 1 FROM pending_signatures s
         WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
       LIMIT ?`,
      [PENDING.waiting, limit]
    )
    const keys = rows
      .map((r: any) => r.release_key as string)
      .filter((key: string) => !explored(key, samplePercent))
    if (keys.length === 0) return 0
    const { rowCount } = await db.rawQuery(
      `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
      [PENDING.discarded, PENDING.waiting, keys]
    )
    return rowCount ?? 0
  }
}

export default new PendingLadder()