Daan MeijerandClaude Fable 5.1
aa4022295a
The tiny-archive rule (dropTinyArchives, both indexers): a settled single archive of one article and 2-100 kB under a release-like name, spaces allowed, outside the exempt groups and without an e-book, subtitle, crack, cover or sidecar word: the 2015-17 exe-stub flood in miniature, 1.66M catalog rows (LOG.md (131), (133)). PendingLadder.discard scanned every waiting row against the dropped signatures and classes, 9+ minutes a sweep inside the claim, stalling every lane: now driven from the dropped classes and signatures into the indexes. The namer feeds the indexer the span's overview text chunk by chunk (one string of it topped V8's limit) and gives up spans over 3M articles. Migration 7000: a partial index on the taken probe candidates for the prober's start. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
431 lines
17 KiB
TypeScript
431 lines
17 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||
import coordinatorConfig from '#config/coordinator'
|
||
import { explored } from '#lib/rules/index'
|
||
import { usableName, type Inspection } from '#lib/probe/index'
|
||
import RuleSets from '#services/rule_sets'
|
||
import { twinKey, twinNames } from '#services/hash_twins'
|
||
|
||
/** pending_releases.status (migration 6400). */
|
||
export const PENDING = {
|
||
waiting: 0,
|
||
named: 1,
|
||
discarded: 2,
|
||
exhausted: 3,
|
||
naming: 4,
|
||
taken: 9,
|
||
} as const
|
||
|
||
/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
|
||
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
|
||
|
||
export interface PendingTarget {
|
||
kind: string
|
||
messageId: string
|
||
file: string
|
||
/** The target article's bytes (candidates from 2026-11-02 on). */
|
||
bytes?: number
|
||
}
|
||
|
||
/** One probe the ladder hands the prober: the next target of a pending release. */
|
||
export interface PendingProbe {
|
||
/** pending_releases.release_key as hex. */
|
||
releaseKey: string
|
||
kind: string
|
||
messageId: string
|
||
file: string
|
||
tool: string
|
||
groupId: number
|
||
/** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
|
||
klass: number
|
||
poster: string
|
||
/** Targets probed before this one. */
|
||
tries: number
|
||
targets: number
|
||
}
|
||
|
||
export type PendingOutcome = {
|
||
probe: PendingProbe
|
||
status: 'done' | 'missing' | 'error' | 'waiting'
|
||
inspection?: Inspection
|
||
}
|
||
|
||
export function byEffort(targets: PendingTarget[]): PendingTarget[] {
|
||
const rank = (kind: string) => {
|
||
const i = (EFFORT as readonly string[]).indexOf(kind)
|
||
return i < 0 ? EFFORT.length : i
|
||
}
|
||
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
|
||
}
|
||
|
||
/** A class's record (pending_classes), for the claim's order. */
|
||
interface ClassRow {
|
||
tool: string
|
||
groupId: number
|
||
klass: number
|
||
probed: number
|
||
named: number
|
||
waiting: number
|
||
}
|
||
|
||
/** The record is read again this often. */
|
||
const CLASSES_MS = 60_000
|
||
/** A class found short of a claim's share is left alone this long. */
|
||
const EMPTY_MS = 5 * 60_000
|
||
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
|
||
const PRIOR_RATE = 0.15
|
||
const PRIOR_WEIGHT = 10
|
||
/** One class takes at most this share of a claim: the rest explores the classes after it. */
|
||
const CLASS_SHARE = 0.25
|
||
|
||
/** The expected naming rate of a class: its record shrunk towards the prior. */
|
||
export function classScore(c: { probed: number; named: number }): number {
|
||
return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
|
||
}
|
||
|
||
/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
|
||
function claimed(rows: any[]): any[] {
|
||
const bytes = (r: any) =>
|
||
r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
|
||
return [...rows].sort(
|
||
(a, b) =>
|
||
bytes(a) - bytes(b) ||
|
||
new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
|
||
Number(a.first_article) - Number(b.first_article)
|
||
)
|
||
}
|
||
|
||
function toProbe(r: any): PendingProbe {
|
||
const targets = byEffort(r.candidates as PendingTarget[])
|
||
const tries = Number(r.tries)
|
||
const target = targets[Math.min(tries, targets.length - 1)]
|
||
return {
|
||
releaseKey: r.release_key,
|
||
kind: target.kind,
|
||
messageId: target.messageId,
|
||
file: target.file,
|
||
tool: r.tool,
|
||
groupId: Number(r.group_id),
|
||
klass: Number(r.klass ?? 0),
|
||
poster: r.poster,
|
||
tries,
|
||
targets: targets.length,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
|
||
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
|
||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
|
||
* class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
|
||
* or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
|
||
* classes with the best record first (classScore), the cheapest target first within one.
|
||
*/
|
||
class PendingLadder {
|
||
/** A previous prober's claims back in the queue. */
|
||
async release(): Promise<number> {
|
||
const { rowCount } = await db.rawQuery(
|
||
`UPDATE pending_releases SET status = ? WHERE status = ?`,
|
||
[PENDING.waiting, PENDING.taken]
|
||
)
|
||
return rowCount ?? 0
|
||
}
|
||
|
||
/** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
|
||
private classes: ClassRow[] = []
|
||
private classesAt = 0
|
||
private empty = new Map<string, number>()
|
||
|
||
/** The record read again on the next claim (the tests). */
|
||
forget() {
|
||
this.classesAt = 0
|
||
this.empty.clear()
|
||
}
|
||
|
||
/** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
|
||
private async classRows(): Promise<ClassRow[]> {
|
||
if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
|
||
const { rows } = await db.rawQuery(
|
||
`SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
|
||
)
|
||
this.classes = rows
|
||
.map((r: any) => ({
|
||
tool: r.tool as string,
|
||
groupId: Number(r.group_id),
|
||
klass: Number(r.klass),
|
||
probed: Number(r.probed),
|
||
named: Number(r.named),
|
||
waiting: Number(r.waiting),
|
||
}))
|
||
.sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
|
||
this.classesAt = Date.now()
|
||
return this.classes
|
||
}
|
||
|
||
/**
|
||
* The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
|
||
* at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
|
||
* and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
|
||
* classes, status 0 of a class without a row).
|
||
*/
|
||
async claim(n: number): Promise<PendingProbe[]> {
|
||
if (n <= 0) return []
|
||
const out: PendingProbe[] = []
|
||
const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
|
||
const now = Date.now()
|
||
for (const c of await this.classRows()) {
|
||
if (out.length >= n) break
|
||
const key = `${c.tool}\t${c.groupId}\t${c.klass}`
|
||
if ((this.empty.get(key) ?? 0) > now) continue
|
||
const want = Math.min(perClass, n - out.length)
|
||
const { rows } = await db.rawQuery(
|
||
`UPDATE pending_releases p SET status = ? FROM (
|
||
SELECT release_key FROM pending_releases
|
||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
|
||
ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||
WHERE p.release_key = c.release_key
|
||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||
[PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
|
||
)
|
||
if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
|
||
if (rows.length > 0) {
|
||
await db.rawQuery(
|
||
`UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||
[rows.length, c.tool, c.groupId, c.klass]
|
||
)
|
||
}
|
||
out.push(...claimed(rows).map(toProbe))
|
||
}
|
||
if (out.length < n) {
|
||
const { rows } = await db.rawQuery(
|
||
`UPDATE pending_releases p SET status = ? FROM (
|
||
SELECT release_key FROM pending_releases
|
||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||
ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||
WHERE p.release_key = c.release_key
|
||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||
[PENDING.taken, PENDING.waiting, n - out.length]
|
||
)
|
||
out.push(...claimed(rows).map(toProbe))
|
||
}
|
||
return out
|
||
}
|
||
|
||
/**
|
||
* What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
|
||
* end of the ladder (status 3); the finished climbs counted on their signatures.
|
||
*/
|
||
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
|
||
if (outcomes.length === 0) return
|
||
// A twin by content names what the probe itself didn't (hash_twins.ts).
|
||
const twins = await twinNames(
|
||
outcomes
|
||
.filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
|
||
.map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
|
||
trx
|
||
)
|
||
const rows = outcomes.map((o) => {
|
||
const p = o.probe
|
||
if (o.status === 'waiting') {
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.waiting,
|
||
tries: p.tries,
|
||
named: null,
|
||
inspection: null,
|
||
final: null,
|
||
}
|
||
}
|
||
const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||
const twin =
|
||
o.status === 'done' && o.inspection?.hash16k
|
||
? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
|
||
: undefined
|
||
const name = probed ?? twin ?? null
|
||
const tries = p.tries + 1
|
||
const inspection = {
|
||
kind: p.kind,
|
||
status: o.status,
|
||
result: o.inspection ? { ...o.inspection, par2: undefined } : null,
|
||
}
|
||
if (name)
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.naming,
|
||
tries: 0,
|
||
named: name,
|
||
inspection,
|
||
final: 'named',
|
||
}
|
||
if (tries >= p.targets)
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.exhausted,
|
||
tries,
|
||
named: null,
|
||
inspection,
|
||
final: 'exhausted',
|
||
}
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.waiting,
|
||
tries,
|
||
named: null,
|
||
inspection,
|
||
final: null,
|
||
}
|
||
})
|
||
await trx.rawQuery(
|
||
`UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
|
||
inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
|
||
probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
|
||
WHERE p.release_key = decode(v.release_key, 'hex')`,
|
||
[JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
|
||
)
|
||
// The signatures: the poster's own and the tool's and group's together.
|
||
const counts = new Map<
|
||
string,
|
||
{ tool: string; group_id: number; poster: string; probed: number; named: number }
|
||
>()
|
||
for (const [i, o] of outcomes.entries()) {
|
||
const final = rows[i].final
|
||
if (!final) continue
|
||
for (const poster of [o.probe.poster, '']) {
|
||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
|
||
const entry = counts.get(key) ?? {
|
||
tool: o.probe.tool,
|
||
group_id: o.probe.groupId,
|
||
poster,
|
||
probed: 0,
|
||
named: 0,
|
||
}
|
||
entry.probed++
|
||
if (final === 'named') entry.named++
|
||
counts.set(key, entry)
|
||
}
|
||
}
|
||
if (counts.size === 0) return
|
||
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
|
||
await trx.rawQuery(
|
||
`INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
|
||
SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
|
||
ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
|
||
[JSON.stringify([...counts.values()])]
|
||
)
|
||
await trx.rawQuery(
|
||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
|
||
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
|
||
[
|
||
pendingMinProbes,
|
||
pendingMaxNamed,
|
||
JSON.stringify(
|
||
[...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
|
||
),
|
||
]
|
||
)
|
||
// The classes: the same record per (tool, group, class), and a probe given back is waiting again.
|
||
const classes = new Map<
|
||
string,
|
||
{
|
||
tool: string
|
||
group_id: number
|
||
klass: number
|
||
probed: number
|
||
named: number
|
||
waiting: number
|
||
}
|
||
>()
|
||
for (const [i, o] of outcomes.entries()) {
|
||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
|
||
const entry = classes.get(key) ?? {
|
||
tool: o.probe.tool,
|
||
group_id: o.probe.groupId,
|
||
klass: o.probe.klass,
|
||
probed: 0,
|
||
named: 0,
|
||
waiting: 0,
|
||
}
|
||
const final = rows[i].final
|
||
if (final) entry.probed++
|
||
if (final === 'named') entry.named++
|
||
if (o.status === 'waiting') entry.waiting++
|
||
classes.set(key, entry)
|
||
}
|
||
await trx.rawQuery(
|
||
`INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
|
||
SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
|
||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
|
||
waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
|
||
[JSON.stringify([...classes.values()])]
|
||
)
|
||
await trx.rawQuery(
|
||
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
|
||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||
[
|
||
pendingMinProbes,
|
||
pendingMaxNamed,
|
||
JSON.stringify(
|
||
[...classes.values()].map(({ tool, group_id, klass }) => ({ tool, group_id, klass }))
|
||
),
|
||
]
|
||
)
|
||
}
|
||
|
||
/**
|
||
* Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
|
||
* many. Driven from the dropped classes and signatures into the claim's index and the ladder's: a scan of the
|
||
* waiting rows testing each (44M on 2026-11-02) ran 9+ minutes on the HDD every sweep (LOG.md (133)).
|
||
*/
|
||
async discard(limit = 10_000): Promise<number> {
|
||
const { samplePercent } = await RuleSets.current()
|
||
const { rows: byClass } = await db.rawQuery(
|
||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_classes c
|
||
JOIN pending_releases p ON p.status = ? AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass
|
||
WHERE c.dropped LIMIT ?`,
|
||
[PENDING.waiting, limit]
|
||
)
|
||
const { rows: bySignature } = await db.rawQuery(
|
||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_signatures s
|
||
JOIN pending_releases p ON p.status = ? AND p.group_id = s.group_id AND p.tool = s.tool AND (s.poster = '' OR s.poster = coalesce(p.poster, ''))
|
||
WHERE s.dropped LIMIT ?`,
|
||
[PENDING.waiting, limit]
|
||
)
|
||
const keys = [
|
||
...new Set([...byClass, ...bySignature].map((r: any) => r.release_key as string)),
|
||
].filter((key: string) => !explored(key, samplePercent))
|
||
if (keys.length === 0) return 0
|
||
const { rows: gone } = await db.rawQuery(
|
||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
|
||
RETURNING tool, group_id, klass`,
|
||
[PENDING.discarded, PENDING.waiting, keys]
|
||
)
|
||
const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
|
||
for (const g of gone) {
|
||
const key = `${g.tool}\t${g.group_id}\t${g.klass}`
|
||
const entry = counts.get(key) ?? {
|
||
tool: g.tool,
|
||
group_id: Number(g.group_id),
|
||
klass: Number(g.klass),
|
||
n: 0,
|
||
}
|
||
entry.n++
|
||
counts.set(key, entry)
|
||
}
|
||
if (counts.size > 0) {
|
||
await db.rawQuery(
|
||
`UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
|
||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||
[JSON.stringify([...counts.values()])]
|
||
)
|
||
}
|
||
return gone.length
|
||
}
|
||
}
|
||
|
||
export default new PendingLadder()
|