Pending ladder by class, twins by content, dropped classes at the edge
The prober names 6% of the unnamed releases it finishes and the pending queue (44M) grows eleven times faster than it probes. A sample of 44,530 finished pending rows shows the headers predict a name: size bucket, name shape, completeness, tool and group separate 0% classes from 95% ones, while the learner keyed on tool and group alone (LOG.md (130)). - lib/release/classes.ts, names::pending_class, SQL pending_class(): a release's class (size × shape × partial, 1..70), the three in step. - Migration 6800: pending_releases.klass and target_bytes (the first target's article; candidates carry bytes now), pending_classes with each class's climbs, names, waiting and verdict, the claim's index. - The ladder claims the classes with the best record first (a prior for the unexplored, at most a quarter of a claim from one), the cheapest target first within one, then the oldest of the rest; records classes like signatures and drops them at the same thresholds; discards their waiting releases. Claimed rows sorted in code: an UPDATE returns none. - The rule set's droppedClasses: the edge keeps unnamed releases of a dropped class out of the pending store (counted as garbage), the explored sample excepted. - hash_twins.ts: a probed file whose first 16 KiB and size match a named catalog release's names a pending release the probe didn't, and sets the inner name of a catalog release without one. - pending:classes classes the rows from before and seeds the record from the finished ones; rules:classes publishes the dropped classes; releases:readable also marks the rows' candidates unreadable. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed
1 parent
81a5dce45a
commit
44c0c73b05
20 files changed
+927
-55
No files matched your search
+226
-33
@@ -4,6 +4,7 @@ import coordinatorConfig from '#config/coordinator'
|
||||
import { explored } from '#lib/rules/index'
|
||||
import { usableName, type Inspection } from '#lib/probe/index'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { twinKey, twinNames } from '#services/hash_twins'
|
||||
|
||||
/** pending_releases.status (migration 6400). */
|
||||
export const PENDING = {
|
||||
@@ -22,6 +23,8 @@ export interface PendingTarget {
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
/** The target article's bytes (candidates from 2026-11-02 on). */
|
||||
bytes?: number
|
||||
}
|
||||
|
||||
/** One probe the ladder hands the prober: the next target of a pending release. */
|
||||
@@ -33,6 +36,8 @@ export interface PendingProbe {
|
||||
file: string
|
||||
tool: string
|
||||
groupId: number
|
||||
/** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
|
||||
klass: number
|
||||
poster: string
|
||||
/** Targets probed before this one. */
|
||||
tries: number
|
||||
@@ -53,12 +58,69 @@ export function byEffort(targets: PendingTarget[]): PendingTarget[] {
|
||||
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
|
||||
}
|
||||
|
||||
/** A class's record (pending_classes), for the claim's order. */
|
||||
interface ClassRow {
|
||||
tool: string
|
||||
groupId: number
|
||||
klass: number
|
||||
probed: number
|
||||
named: number
|
||||
waiting: number
|
||||
}
|
||||
|
||||
/** The record is read again this often. */
|
||||
const CLASSES_MS = 60_000
|
||||
/** A class found short of a claim's share is left alone this long. */
|
||||
const EMPTY_MS = 5 * 60_000
|
||||
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
|
||||
const PRIOR_RATE = 0.15
|
||||
const PRIOR_WEIGHT = 10
|
||||
/** One class takes at most this share of a claim: the rest explores the classes after it. */
|
||||
const CLASS_SHARE = 0.25
|
||||
|
||||
/** The expected naming rate of a class: its record shrunk towards the prior. */
|
||||
export function classScore(c: { probed: number; named: number }): number {
|
||||
return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
|
||||
}
|
||||
|
||||
/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
|
||||
function claimed(rows: any[]): any[] {
|
||||
const bytes = (r: any) =>
|
||||
r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
|
||||
return [...rows].sort(
|
||||
(a, b) =>
|
||||
bytes(a) - bytes(b) ||
|
||||
new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
|
||||
Number(a.first_article) - Number(b.first_article)
|
||||
)
|
||||
}
|
||||
|
||||
function toProbe(r: any): PendingProbe {
|
||||
const targets = byEffort(r.candidates as PendingTarget[])
|
||||
const tries = Number(r.tries)
|
||||
const target = targets[Math.min(tries, targets.length - 1)]
|
||||
return {
|
||||
releaseKey: r.release_key,
|
||||
kind: target.kind,
|
||||
messageId: target.messageId,
|
||||
file: target.file,
|
||||
tool: r.tool,
|
||||
groupId: Number(r.group_id),
|
||||
klass: Number(r.klass ?? 0),
|
||||
poster: r.poster,
|
||||
tries,
|
||||
targets: targets.length,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
|
||||
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
|
||||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
|
||||
* signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
|
||||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
|
||||
* class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
|
||||
* or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
|
||||
* classes with the best record first (classScore), the cheapest target first within one.
|
||||
*/
|
||||
class PendingLadder {
|
||||
/** A previous prober's claims back in the queue. */
|
||||
@@ -70,34 +132,84 @@ class PendingLadder {
|
||||
return rowCount ?? 0
|
||||
}
|
||||
|
||||
/** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
|
||||
/** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
|
||||
private classes: ClassRow[] = []
|
||||
private classesAt = 0
|
||||
private empty = new Map<string, number>()
|
||||
|
||||
/** The record read again on the next claim (the tests). */
|
||||
forget() {
|
||||
this.classesAt = 0
|
||||
this.empty.clear()
|
||||
}
|
||||
|
||||
/** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
|
||||
private async classRows(): Promise<ClassRow[]> {
|
||||
if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
|
||||
)
|
||||
this.classes = rows
|
||||
.map((r: any) => ({
|
||||
tool: r.tool as string,
|
||||
groupId: Number(r.group_id),
|
||||
klass: Number(r.klass),
|
||||
probed: Number(r.probed),
|
||||
named: Number(r.named),
|
||||
waiting: Number(r.waiting),
|
||||
}))
|
||||
.sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
|
||||
this.classesAt = Date.now()
|
||||
return this.classes
|
||||
}
|
||||
|
||||
/**
|
||||
* The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
|
||||
* at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
|
||||
* and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
|
||||
* classes, status 0 of a class without a row).
|
||||
*/
|
||||
async claim(n: number): Promise<PendingProbe[]> {
|
||||
if (n <= 0) return []
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
|
||||
[PENDING.taken, PENDING.waiting, n]
|
||||
)
|
||||
return rows.map((r: any) => {
|
||||
const targets = byEffort(r.candidates as PendingTarget[])
|
||||
const tries = Number(r.tries)
|
||||
const target = targets[Math.min(tries, targets.length - 1)]
|
||||
return {
|
||||
releaseKey: r.release_key,
|
||||
kind: target.kind,
|
||||
messageId: target.messageId,
|
||||
file: target.file,
|
||||
tool: r.tool,
|
||||
groupId: Number(r.group_id),
|
||||
poster: r.poster,
|
||||
tries,
|
||||
targets: targets.length,
|
||||
const out: PendingProbe[] = []
|
||||
const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
|
||||
const now = Date.now()
|
||||
for (const c of await this.classRows()) {
|
||||
if (out.length >= n) break
|
||||
const key = `${c.tool}\t${c.groupId}\t${c.klass}`
|
||||
if ((this.empty.get(key) ?? 0) > now) continue
|
||||
const want = Math.min(perClass, n - out.length)
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||||
[PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
|
||||
)
|
||||
if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
|
||||
if (rows.length > 0) {
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||||
[rows.length, c.tool, c.groupId, c.klass]
|
||||
)
|
||||
}
|
||||
})
|
||||
out.push(...claimed(rows).map(toProbe))
|
||||
}
|
||||
if (out.length < n) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||||
[PENDING.taken, PENDING.waiting, n - out.length]
|
||||
)
|
||||
out.push(...claimed(rows).map(toProbe))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -106,6 +218,13 @@ class PendingLadder {
|
||||
*/
|
||||
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
|
||||
if (outcomes.length === 0) return
|
||||
// A twin by content names what the probe itself didn't (hash_twins.ts).
|
||||
const twins = await twinNames(
|
||||
outcomes
|
||||
.filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
|
||||
.map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
|
||||
trx
|
||||
)
|
||||
const rows = outcomes.map((o) => {
|
||||
const p = o.probe
|
||||
if (o.status === 'waiting') {
|
||||
@@ -118,7 +237,12 @@ class PendingLadder {
|
||||
final: null,
|
||||
}
|
||||
}
|
||||
const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||||
const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||||
const twin =
|
||||
o.status === 'done' && o.inspection?.hash16k
|
||||
? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
|
||||
: undefined
|
||||
const name = probed ?? twin ?? null
|
||||
const tries = p.tries + 1
|
||||
const inspection = {
|
||||
kind: p.kind,
|
||||
@@ -202,6 +326,53 @@ class PendingLadder {
|
||||
),
|
||||
]
|
||||
)
|
||||
// The classes: the same record per (tool, group, class), and a probe given back is waiting again.
|
||||
const classes = new Map<
|
||||
string,
|
||||
{
|
||||
tool: string
|
||||
group_id: number
|
||||
klass: number
|
||||
probed: number
|
||||
named: number
|
||||
waiting: number
|
||||
}
|
||||
>()
|
||||
for (const [i, o] of outcomes.entries()) {
|
||||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
|
||||
const entry = classes.get(key) ?? {
|
||||
tool: o.probe.tool,
|
||||
group_id: o.probe.groupId,
|
||||
klass: o.probe.klass,
|
||||
probed: 0,
|
||||
named: 0,
|
||||
waiting: 0,
|
||||
}
|
||||
const final = rows[i].final
|
||||
if (final) entry.probed++
|
||||
if (final === 'named') entry.named++
|
||||
if (o.status === 'waiting') entry.waiting++
|
||||
classes.set(key, entry)
|
||||
}
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
|
||||
SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
|
||||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
|
||||
waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
|
||||
[JSON.stringify([...classes.values()])]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
|
||||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||||
[
|
||||
pendingMinProbes,
|
||||
pendingMaxNamed,
|
||||
JSON.stringify(
|
||||
[...classes.values()].map(({ tool, group_id, klass }) => ({ tool, group_id, klass }))
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
/** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
|
||||
@@ -209,9 +380,10 @@ class PendingLadder {
|
||||
const { samplePercent } = await RuleSets.current()
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
|
||||
WHERE p.status = ? AND EXISTS (
|
||||
WHERE p.status = ? AND (EXISTS (
|
||||
SELECT 1 FROM pending_signatures s
|
||||
WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
|
||||
OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass))
|
||||
LIMIT ?`,
|
||||
[PENDING.waiting, limit]
|
||||
)
|
||||
@@ -219,11 +391,32 @@ class PendingLadder {
|
||||
.map((r: any) => r.release_key as string)
|
||||
.filter((key: string) => !explored(key, samplePercent))
|
||||
if (keys.length === 0) return 0
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
|
||||
const { rows: gone } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
|
||||
RETURNING tool, group_id, klass`,
|
||||
[PENDING.discarded, PENDING.waiting, keys]
|
||||
)
|
||||
return rowCount ?? 0
|
||||
const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
|
||||
for (const g of gone) {
|
||||
const key = `${g.tool}\t${g.group_id}\t${g.klass}`
|
||||
const entry = counts.get(key) ?? {
|
||||
tool: g.tool,
|
||||
group_id: Number(g.group_id),
|
||||
klass: Number(g.klass),
|
||||
n: 0,
|
||||
}
|
||||
entry.n++
|
||||
counts.set(key, entry)
|
||||
}
|
||||
if (counts.size > 0) {
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
|
||||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||||
[JSON.stringify([...counts.values()])]
|
||||
)
|
||||
}
|
||||
return gone.length
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user