daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pending ladder by class, twins by content, dropped classes at the edge
Test and deploy / test (push) Successful in 3m9s
Details
Test and deploy / deploy (push) Successful in 40m8s
Details

Browse Source 
The prober names 6% of the unnamed releases it finishes and the pending
queue (44M) grows eleven times faster than it probes. A sample of 44,530
finished pending rows shows the headers predict a name: size bucket, name
shape, completeness, tool and group separate 0% classes from 95% ones,
while the learner keyed on tool and group alone (LOG.md (130)).

- lib/release/classes.ts, names::pending_class, SQL pending_class():
  a release's class (size × shape × partial, 1..70), the three in step.
- Migration 6800: pending_releases.klass and target_bytes (the first
  target's article; candidates carry bytes now), pending_classes with
  each class's climbs, names, waiting and verdict, the claim's index.
- The ladder claims the classes with the best record first (a prior for
  the unexplored, at most a quarter of a claim from one), the cheapest
  target first within one, then the oldest of the rest; records classes
  like signatures and drops them at the same thresholds; discards their
  waiting releases. Claimed rows sorted in code: an UPDATE returns none.
- The rule set's droppedClasses: the edge keeps unnamed releases of a
  dropped class out of the pending store (counted as garbage), the
  explored sample excepted.
- hash_twins.ts: a probed file whose first 16 KiB and size match a named
  catalog release's names a pending release the probe didn't, and sets
  the inner name of a catalog release without one.
- pending:classes classes the rows from before and seeds the record from
  the finished ones; rules:classes publishes the dropped classes;
  releases:readable also marks the rows' candidates unreadable.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-11-02 12:22:32 +01:00
1 parent 81a5dce45a
commit 44c0c73b05
20 files changed
+927 -55
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/batch_store.ts
+32 -7
View File
@@ -9,6 +9,7 @@ import { SIDECAR } from '#services/sidecar_folder'
import { identityKey, releaseKey } from '#lib/release/keys'
import { sourceCode } from '#lib/release/source'
import { uploadTool } from '#lib/rules/index'
import { byEffort } from '#services/pending_ladder'
import { TEXT_KINDS, TEXT_MAX, textsToStore } from '#lib/release/text_kind'
/**
@@ -179,11 +180,14 @@ class BatchStore {
candidates: ProbeCandidate[]
): Promise<number> {
if (releases.length === 0) return 0
const targets = new Map<string, { kind: string; messageId: string; file: string }[]>()
const targets = new Map<
string,
{ kind: string; messageId: string; file: string; bytes?: number }[]
>()
for (const c of candidates) {
const key = slot(c.releaseKey, c.upload ?? 0)
const list = targets.get(key) ?? []
list.push({ kind: c.kind, messageId: c.messageId, file: c.file })
list.push({ kind: c.kind, messageId: c.messageId, file: c.file, bytes: c.bytes })
targets.set(key, list)
}
let written = 0
@@ -207,25 +211,46 @@ class BatchStore {
file_total: r.fileTotal,
tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
candidates: probes,
// The first target's article (by effort): the cheapest probe first within a class (LOG.md (130)).
target_bytes: byEffort(probes)[0]?.bytes ?? null,
}
})
const { rows: done } = await trx.rawQuery(
`INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
files, segments, bytes, partial, finished, file_total, tool, candidates)
files, segments, bytes, partial, finished, file_total, tool, candidates, klass, target_bytes)
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates,
pending_class(v.name, v.bytes, v.partial), v.target_bytes
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb)
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb, target_bytes bigint)
ON CONFLICT (release_key) DO UPDATE SET
files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END,
klass = pending_class(pending_releases.name, pending_releases.bytes + excluded.bytes, excluded.partial),
target_bytes = coalesce(pending_releases.target_bytes, excluded.target_bytes)
WHERE NOT pending_releases.finished
RETURNING release_key`,
RETURNING release_key, tool, klass, (xmax = 0) AS inserted`,
[groupId, JSON.stringify(rows)]
)
written += done.length
// Each class seen gets its row in the ladder's record, so the claim explores it (pending_classes).
const classes = new Map<string, { tool: string; klass: number; waiting: number }>()
for (const d of done) {
const key = `${d.tool}\t${d.klass}`
const entry = classes.get(key) ?? { tool: d.tool, klass: Number(d.klass), waiting: 0 }
if (d.inserted) entry.waiting++
classes.set(key, entry)
}
if (classes.size > 0) {
await trx.rawQuery(
`INSERT INTO pending_classes (tool, group_id, klass, waiting)
SELECT v.tool, ?, v.klass, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, klass int, waiting int)
ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = pending_classes.waiting + excluded.waiting`,
[groupId, JSON.stringify([...classes.values()])]
)
}
}
return written
}
app/services/hash_twins.ts
+27
View File
@@ -0,0 +1,27 @@
import db from '@adonisjs/lucid/services/db'
/**
* Twins by content (LOG.md (130)): a probed file's first 16 KiB and size (file_hashes, what the par2 merge and the
* repeated verdict match) in a catalog release that has a name, readable or found inside. The same film posted
* under scrambles by other tools gets that name without another fetch. Releases with a junk verdict don't count.
*/
export async function twinNames(
pairs: { hash16k: string; size: number }[],
trx: { rawQuery: typeof db.rawQuery } = db
): Promise<Map<string, string>> {
const out = new Map<string, string>()
if (pairs.length === 0) return out
const { rows } = await trx.rawQuery(
`SELECT DISTINCT ON (h.hash16k, h.size) h.hash16k, h.size, coalesce(r.inner_name, r.name) AS name
FROM jsonb_to_recordset(?::jsonb) AS v(hash16k text, size bigint)
JOIN file_hashes h ON h.hash16k = v.hash16k AND h.size = v.size
JOIN releases r ON r.id = h.release_id AND (r.readable OR r.inner_name IS NOT NULL)
WHERE NOT EXISTS (SELECT 1 FROM release_verdicts x WHERE x.release_id = r.id)
ORDER BY h.hash16k, h.size, r.readable DESC, r.id`,
[JSON.stringify(pairs)]
)
for (const r of rows) out.set(`${r.hash16k}\t${r.size}`, r.name)
return out
}
export const twinKey = (hash16k: string, size: number) => `${hash16k}\t${size}`
app/services/pending_ladder.ts
+226 -33
View File
@@ -4,6 +4,7 @@ import coordinatorConfig from '#config/coordinator'
import { explored } from '#lib/rules/index'
import { usableName, type Inspection } from '#lib/probe/index'
import RuleSets from '#services/rule_sets'
import { twinKey, twinNames } from '#services/hash_twins'
/** pending_releases.status (migration 6400). */
export const PENDING = {
@@ -22,6 +23,8 @@ export interface PendingTarget {
kind: string
messageId: string
file: string
/** The target article's bytes (candidates from 2026-11-02 on). */
bytes?: number
}
/** One probe the ladder hands the prober: the next target of a pending release. */
@@ -33,6 +36,8 @@ export interface PendingProbe {
file: string
tool: string
groupId: number
/** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
klass: number
poster: string
/** Targets probed before this one. */
tries: number
@@ -53,12 +58,69 @@ export function byEffort(targets: PendingTarget[]): PendingTarget[] {
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
}
/** A class's record (pending_classes), for the claim's order. */
interface ClassRow {
tool: string
groupId: number
klass: number
probed: number
named: number
waiting: number
}
/** The record is read again this often. */
const CLASSES_MS = 60_000
/** A class found short of a claim's share is left alone this long. */
const EMPTY_MS = 5 * 60_000
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
const PRIOR_RATE = 0.15
const PRIOR_WEIGHT = 10
/** One class takes at most this share of a claim: the rest explores the classes after it. */
const CLASS_SHARE = 0.25
/** The expected naming rate of a class: its record shrunk towards the prior. */
export function classScore(c: { probed: number; named: number }): number {
return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
}
/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
function claimed(rows: any[]): any[] {
const bytes = (r: any) =>
r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
return [...rows].sort(
(a, b) =>
bytes(a) - bytes(b) ||
new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
Number(a.first_article) - Number(b.first_article)
)
}
function toProbe(r: any): PendingProbe {
const targets = byEffort(r.candidates as PendingTarget[])
const tries = Number(r.tries)
const target = targets[Math.min(tries, targets.length - 1)]
return {
releaseKey: r.release_key,
kind: target.kind,
messageId: target.messageId,
file: target.file,
tool: r.tool,
groupId: Number(r.group_id),
klass: Number(r.klass ?? 0),
poster: r.poster,
tries,
targets: targets.length,
}
}
/**
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
* every finished climb counted on its signature (tool, group, poster; and the tool and group together), and a
* signature with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured.
* every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
* class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
* or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
* classes with the best record first (classScore), the cheapest target first within one.
*/
class PendingLadder {
/** A previous prober's claims back in the queue. */
@@ -70,34 +132,84 @@ class PendingLadder {
return rowCount ?? 0
}
/** The next target of up to `n` waiting releases with a target left, oldest first, claimed. */
/** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
private classes: ClassRow[] = []
private classesAt = 0
private empty = new Map<string, number>()
/** The record read again on the next claim (the tests). */
forget() {
this.classesAt = 0
this.empty.clear()
}
/** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
private async classRows(): Promise<ClassRow[]> {
if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
const { rows } = await db.rawQuery(
`SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
)
this.classes = rows
.map((r: any) => ({
tool: r.tool as string,
groupId: Number(r.group_id),
klass: Number(r.klass),
probed: Number(r.probed),
named: Number(r.named),
waiting: Number(r.waiting),
}))
.sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
this.classesAt = Date.now()
return this.classes
}
/**
* The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
* at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
* and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
* classes, status 0 of a class without a row).
*/
async claim(n: number): Promise<PendingProbe[]> {
if (n <= 0) return []
const { rows } = await db.rawQuery(
`UPDATE pending_releases p SET status = ? FROM (
SELECT release_key FROM pending_releases
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
ORDER BY created_at LIMIT ? FOR UPDATE SKIP LOCKED) c
WHERE p.release_key = c.release_key
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, coalesce(p.poster, '') AS poster`,
[PENDING.taken, PENDING.waiting, n]
)
return rows.map((r: any) => {
const targets = byEffort(r.candidates as PendingTarget[])
const tries = Number(r.tries)
const target = targets[Math.min(tries, targets.length - 1)]
return {
releaseKey: r.release_key,
kind: target.kind,
messageId: target.messageId,
file: target.file,
tool: r.tool,
groupId: Number(r.group_id),
poster: r.poster,
tries,
targets: targets.length,
const out: PendingProbe[] = []
const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
const now = Date.now()
for (const c of await this.classRows()) {
if (out.length >= n) break
const key = `${c.tool}\t${c.groupId}\t${c.klass}`
if ((this.empty.get(key) ?? 0) > now) continue
const want = Math.min(perClass, n - out.length)
const { rows } = await db.rawQuery(
`UPDATE pending_releases p SET status = ? FROM (
SELECT release_key FROM pending_releases
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
WHERE p.release_key = c.release_key
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
[PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
)
if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
if (rows.length > 0) {
await db.rawQuery(
`UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
[rows.length, c.tool, c.groupId, c.klass]
)
}
})
out.push(...claimed(rows).map(toProbe))
}
if (out.length < n) {
const { rows } = await db.rawQuery(
`UPDATE pending_releases p SET status = ? FROM (
SELECT release_key FROM pending_releases
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
WHERE p.release_key = c.release_key
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
[PENDING.taken, PENDING.waiting, n - out.length]
)
out.push(...claimed(rows).map(toProbe))
}
return out
}
/**
@@ -106,6 +218,13 @@ class PendingLadder {
*/
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
if (outcomes.length === 0) return
// A twin by content names what the probe itself didn't (hash_twins.ts).
const twins = await twinNames(
outcomes
.filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
.map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
trx
)
const rows = outcomes.map((o) => {
const p = o.probe
if (o.status === 'waiting') {
@@ -118,7 +237,12 @@ class PendingLadder {
final: null,
}
}
const name = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
const twin =
o.status === 'done' && o.inspection?.hash16k
? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
: undefined
const name = probed ?? twin ?? null
const tries = p.tries + 1
const inspection = {
kind: p.kind,
@@ -202,6 +326,53 @@ class PendingLadder {
),
]
)
// The classes: the same record per (tool, group, class), and a probe given back is waiting again.
const classes = new Map<
string,
{
tool: string
group_id: number
klass: number
probed: number
named: number
waiting: number
}
>()
for (const [i, o] of outcomes.entries()) {
const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
const entry = classes.get(key) ?? {
tool: o.probe.tool,
group_id: o.probe.groupId,
klass: o.probe.klass,
probed: 0,
named: 0,
waiting: 0,
}
const final = rows[i].final
if (final) entry.probed++
if (final === 'named') entry.named++
if (o.status === 'waiting') entry.waiting++
classes.set(key, entry)
}
await trx.rawQuery(
`INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
[JSON.stringify([...classes.values()])]
)
await trx.rawQuery(
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
[
pendingMinProbes,
pendingMaxNamed,
JSON.stringify(
[...classes.values()].map(({ tool, group_id, klass }) => ({ tool, group_id, klass }))
),
]
)
}
/** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
@@ -209,9 +380,10 @@ class PendingLadder {
const { samplePercent } = await RuleSets.current()
const { rows } = await db.rawQuery(
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
WHERE p.status = ? AND EXISTS (
WHERE p.status = ? AND (EXISTS (
SELECT 1 FROM pending_signatures s
WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass))
LIMIT ?`,
[PENDING.waiting, limit]
)
@@ -219,11 +391,32 @@ class PendingLadder {
.map((r: any) => r.release_key as string)
.filter((key: string) => !explored(key, samplePercent))
if (keys.length === 0) return 0
const { rowCount } = await db.rawQuery(
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)`,
const { rows: gone } = await db.rawQuery(
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
RETURNING tool, group_id, klass`,
[PENDING.discarded, PENDING.waiting, keys]
)
return rowCount ?? 0
const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
for (const g of gone) {
const key = `${g.tool}\t${g.group_id}\t${g.klass}`
const entry = counts.get(key) ?? {
tool: g.tool,
group_id: Number(g.group_id),
klass: Number(g.klass),
n: 0,
}
entry.n++
counts.set(key, entry)
}
if (counts.size > 0) {
await db.rawQuery(
`UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
[JSON.stringify([...counts.values()])]
)
}
return gone.length
}
}
app/services/prober.ts
+12 -1
View File
@@ -9,6 +9,7 @@ import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/i
import { NntpShare } from '#services/nntp_shares'
import { DiskSpace } from '#services/disk_space'
import { decidePar2Completeness } from '#services/par2_completeness'
import { twinKey, twinNames } from '#services/hash_twins'
import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
import { PendingNamer } from '#services/pending_namer'
@@ -392,10 +393,20 @@ export class Prober {
entry.all.push({ kind: r.kind, result: r.result })
byRelease.set(id, entry)
}
// A twin by content names a release its own inspections didn't (hash_twins.ts; LOG.md (130)).
const twins = await twinNames(
done.filter((o) => o.inspection!.hash16k && o.inspection!.size).map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
trx
)
const twinOf = new Map<number, string>()
for (const o of done) {
const twin = o.inspection!.hash16k ? twins.get(twinKey(o.inspection!.hash16k, o.inspection!.size)) : undefined
if (twin && twin !== byRelease.get(o.candidate.releaseId)?.name) twinOf.set(o.candidate.releaseId, twin)
}
const updates = [...byRelease].map(([id, { name, all }]) => ({
id,
passworded: all.some((i) => i.result.passworded === true),
inner_name: innerName(all, name),
inner_name: innerName(all, name) ?? twinOf.get(id) ?? null,
}))
await trx.rawQuery(
`UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()