Daan MeijerandClaude Fable 5.1
411c36eec0
A poster of 2.7 kB broke the index build (a btree key is at most a third of a page) and left an invalid index that IF NOT EXISTS would have taken for the real one: the migration drops it first and indexes md5(poster), and the sweep asks by it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
503 lines
20 KiB
TypeScript
503 lines
20 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||
import coordinatorConfig from '#config/coordinator'
|
||
import { explored } from '#lib/rules/index'
|
||
import { usableName, type Inspection } from '#lib/probe/index'
|
||
import RuleSets from '#services/rule_sets'
|
||
import { twinKey, twinNames } from '#services/hash_twins'
|
||
|
||
/** pending_releases.status (migration 6400). */
|
||
export const PENDING = {
|
||
waiting: 0,
|
||
named: 1,
|
||
discarded: 2,
|
||
exhausted: 3,
|
||
naming: 4,
|
||
taken: 9,
|
||
} as const
|
||
|
||
/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
|
||
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
|
||
|
||
export interface PendingTarget {
|
||
kind: string
|
||
messageId: string
|
||
file: string
|
||
/** The target article's bytes (candidates from 2026-11-02 on). */
|
||
bytes?: number
|
||
}
|
||
|
||
/** One probe the ladder hands the prober: the next target of a pending release. */
|
||
export interface PendingProbe {
|
||
/** pending_releases.release_key as hex. */
|
||
releaseKey: string
|
||
kind: string
|
||
messageId: string
|
||
file: string
|
||
tool: string
|
||
groupId: number
|
||
/** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
|
||
klass: number
|
||
poster: string
|
||
/** Targets probed before this one. */
|
||
tries: number
|
||
targets: number
|
||
}
|
||
|
||
export type PendingOutcome = {
|
||
probe: PendingProbe
|
||
status: 'done' | 'missing' | 'error' | 'waiting'
|
||
inspection?: Inspection
|
||
}
|
||
|
||
export function byEffort(targets: PendingTarget[]): PendingTarget[] {
|
||
const rank = (kind: string) => {
|
||
const i = (EFFORT as readonly string[]).indexOf(kind)
|
||
return i < 0 ? EFFORT.length : i
|
||
}
|
||
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
|
||
}
|
||
|
||
/** A class's record (pending_classes), for the claim's order. */
|
||
interface ClassRow {
|
||
tool: string
|
||
groupId: number
|
||
klass: number
|
||
probed: number
|
||
named: number
|
||
waiting: number
|
||
}
|
||
|
||
/** The record is read again this often. */
|
||
const CLASSES_MS = 60_000
|
||
/** A class found short of a claim's share is left alone this long. */
|
||
const EMPTY_MS = 5 * 60_000
|
||
/**
|
||
* Rows in one order for a statement that locks many of them (pending_releases by key, pending_classes and
|
||
* pending_signatures by theirs): the batch store, the record, the claim and the sweep all lock the same way, so two
|
||
* of them can't deadlock on each other (LOG.md (134)).
|
||
*/
|
||
export function lockOrder<T>(rows: Iterable<T>, key: (row: T) => string): T[] {
|
||
return [...rows].sort((a, b) => {
|
||
const ka = key(a)
|
||
const kb = key(b)
|
||
return ka < kb ? -1 : ka > kb ? 1 : 0
|
||
})
|
||
}
|
||
|
||
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
|
||
const PRIOR_RATE = 0.15
|
||
const PRIOR_WEIGHT = 10
|
||
/** One class takes at most this share of a claim: the rest explores the classes after it. */
|
||
const CLASS_SHARE = 0.25
|
||
|
||
/** The expected naming rate of a class: its record shrunk towards the prior. */
|
||
export function classScore(c: { probed: number; named: number }): number {
|
||
return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
|
||
}
|
||
|
||
/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
|
||
function claimed(rows: any[]): any[] {
|
||
const bytes = (r: any) =>
|
||
r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
|
||
return [...rows].sort(
|
||
(a, b) =>
|
||
bytes(a) - bytes(b) ||
|
||
new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
|
||
Number(a.first_article) - Number(b.first_article)
|
||
)
|
||
}
|
||
|
||
function toProbe(r: any): PendingProbe {
|
||
const targets = byEffort(r.candidates as PendingTarget[])
|
||
const tries = Number(r.tries)
|
||
const target = targets[Math.min(tries, targets.length - 1)]
|
||
return {
|
||
releaseKey: r.release_key,
|
||
kind: target.kind,
|
||
messageId: target.messageId,
|
||
file: target.file,
|
||
tool: r.tool,
|
||
groupId: Number(r.group_id),
|
||
klass: Number(r.klass ?? 0),
|
||
poster: r.poster,
|
||
tries,
|
||
targets: targets.length,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
|
||
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
|
||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
|
||
* class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
|
||
* or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
|
||
* classes with the best record first (classScore), the cheapest target first within one.
|
||
*/
|
||
class PendingLadder {
|
||
/** A previous prober's claims back in the queue. */
|
||
async release(): Promise<number> {
|
||
const { rowCount } = await db.rawQuery(
|
||
`UPDATE pending_releases SET status = ? WHERE status = ?`,
|
||
[PENDING.waiting, PENDING.taken]
|
||
)
|
||
return rowCount ?? 0
|
||
}
|
||
|
||
/** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
|
||
private classes: ClassRow[] = []
|
||
private classesAt = 0
|
||
private empty = new Map<string, number>()
|
||
|
||
/** The record read again on the next claim (the tests). */
|
||
forget() {
|
||
this.classesAt = 0
|
||
this.empty.clear()
|
||
}
|
||
|
||
/** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
|
||
private async classRows(): Promise<ClassRow[]> {
|
||
if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
|
||
const { rows } = await db.rawQuery(
|
||
`SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
|
||
)
|
||
this.classes = rows
|
||
.map((r: any) => ({
|
||
tool: r.tool as string,
|
||
groupId: Number(r.group_id),
|
||
klass: Number(r.klass),
|
||
probed: Number(r.probed),
|
||
named: Number(r.named),
|
||
waiting: Number(r.waiting),
|
||
}))
|
||
.sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
|
||
this.classesAt = Date.now()
|
||
return this.classes
|
||
}
|
||
|
||
/**
|
||
* The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
|
||
* at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
|
||
* and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
|
||
* classes, status 0 of a class without a row).
|
||
*/
|
||
async claim(n: number): Promise<PendingProbe[]> {
|
||
if (n <= 0) return []
|
||
const out: PendingProbe[] = []
|
||
const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
|
||
const now = Date.now()
|
||
for (const c of await this.classRows()) {
|
||
if (out.length >= n) break
|
||
const key = `${c.tool}\t${c.groupId}\t${c.klass}`
|
||
if ((this.empty.get(key) ?? 0) > now) continue
|
||
const want = Math.min(perClass, n - out.length)
|
||
const { rows } = await db.rawQuery(
|
||
`UPDATE pending_releases p SET status = ? FROM (
|
||
SELECT release_key FROM pending_releases
|
||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
|
||
ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||
WHERE p.release_key = c.release_key
|
||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||
[PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
|
||
)
|
||
if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
|
||
if (rows.length > 0) {
|
||
await db.rawQuery(
|
||
`UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||
[rows.length, c.tool, c.groupId, c.klass]
|
||
)
|
||
}
|
||
out.push(...claimed(rows).map(toProbe))
|
||
}
|
||
if (out.length < n) {
|
||
const { rows } = await db.rawQuery(
|
||
`UPDATE pending_releases p SET status = ? FROM (
|
||
SELECT release_key FROM pending_releases
|
||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||
ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||
WHERE p.release_key = c.release_key
|
||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||
[PENDING.taken, PENDING.waiting, n - out.length]
|
||
)
|
||
out.push(...claimed(rows).map(toProbe))
|
||
}
|
||
return out
|
||
}
|
||
|
||
/**
|
||
* What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
|
||
* end of the ladder (status 3); the finished climbs counted on their signatures.
|
||
*/
|
||
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
|
||
if (outcomes.length === 0) return
|
||
// A twin by content names what the probe itself didn't (hash_twins.ts).
|
||
const twins = await twinNames(
|
||
outcomes
|
||
.filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
|
||
.map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
|
||
trx
|
||
)
|
||
const rows = outcomes.map((o) => {
|
||
const p = o.probe
|
||
if (o.status === 'waiting') {
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.waiting,
|
||
tries: p.tries,
|
||
named: null,
|
||
inspection: null,
|
||
final: null,
|
||
}
|
||
}
|
||
const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||
const twin =
|
||
o.status === 'done' && o.inspection?.hash16k
|
||
? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
|
||
: undefined
|
||
const name = probed ?? twin ?? null
|
||
const tries = p.tries + 1
|
||
const inspection = {
|
||
kind: p.kind,
|
||
status: o.status,
|
||
result: o.inspection ? { ...o.inspection, par2: undefined } : null,
|
||
}
|
||
if (name)
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.naming,
|
||
tries: 0,
|
||
named: name,
|
||
inspection,
|
||
final: 'named',
|
||
}
|
||
if (tries >= p.targets)
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.exhausted,
|
||
tries,
|
||
named: null,
|
||
inspection,
|
||
final: 'exhausted',
|
||
}
|
||
return {
|
||
release_key: p.releaseKey,
|
||
status: PENDING.waiting,
|
||
tries,
|
||
named: null,
|
||
inspection,
|
||
final: null,
|
||
}
|
||
})
|
||
await trx.rawQuery(
|
||
`UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
|
||
inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
|
||
probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
|
||
WHERE p.release_key = decode(v.release_key, 'hex')`,
|
||
[
|
||
JSON.stringify(
|
||
lockOrder(
|
||
rows.map(({ final: _final, ...row }) => row),
|
||
(r) => r.release_key
|
||
)
|
||
),
|
||
]
|
||
)
|
||
// The signatures: the poster's own and the tool's and group's together.
|
||
const counts = new Map<
|
||
string,
|
||
{ tool: string; group_id: number; poster: string; probed: number; named: number }
|
||
>()
|
||
for (const [i, o] of outcomes.entries()) {
|
||
const final = rows[i].final
|
||
if (!final) continue
|
||
for (const poster of [o.probe.poster, '']) {
|
||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
|
||
const entry = counts.get(key) ?? {
|
||
tool: o.probe.tool,
|
||
group_id: o.probe.groupId,
|
||
poster,
|
||
probed: 0,
|
||
named: 0,
|
||
}
|
||
entry.probed++
|
||
if (final === 'named') entry.named++
|
||
counts.set(key, entry)
|
||
}
|
||
}
|
||
if (counts.size === 0) return
|
||
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
|
||
await trx.rawQuery(
|
||
`INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
|
||
SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
|
||
ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
|
||
[JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`))]
|
||
)
|
||
await trx.rawQuery(
|
||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float),
|
||
swept_at = CASE WHEN s.dropped THEN s.swept_at END
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
|
||
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
|
||
[
|
||
pendingMinProbes,
|
||
pendingMaxNamed,
|
||
JSON.stringify(
|
||
lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.poster}`).map(
|
||
({ tool, group_id, poster }) => ({ tool, group_id, poster })
|
||
)
|
||
),
|
||
]
|
||
)
|
||
// The classes: the same record per (tool, group, class), and a probe given back is waiting again.
|
||
const classes = new Map<
|
||
string,
|
||
{
|
||
tool: string
|
||
group_id: number
|
||
klass: number
|
||
probed: number
|
||
named: number
|
||
waiting: number
|
||
}
|
||
>()
|
||
for (const [i, o] of outcomes.entries()) {
|
||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
|
||
const entry = classes.get(key) ?? {
|
||
tool: o.probe.tool,
|
||
group_id: o.probe.groupId,
|
||
klass: o.probe.klass,
|
||
probed: 0,
|
||
named: 0,
|
||
waiting: 0,
|
||
}
|
||
const final = rows[i].final
|
||
if (final) entry.probed++
|
||
if (final === 'named') entry.named++
|
||
if (o.status === 'waiting') entry.waiting++
|
||
classes.set(key, entry)
|
||
}
|
||
await trx.rawQuery(
|
||
`INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
|
||
SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
|
||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
|
||
waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
|
||
[JSON.stringify(lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
|
||
)
|
||
await trx.rawQuery(
|
||
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float),
|
||
swept_at = CASE WHEN c.dropped THEN c.swept_at END
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
|
||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||
[
|
||
pendingMinProbes,
|
||
pendingMaxNamed,
|
||
JSON.stringify(
|
||
lockOrder(classes.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`).map(
|
||
({ tool, group_id, klass }) => ({ tool, group_id, klass })
|
||
)
|
||
),
|
||
]
|
||
)
|
||
}
|
||
|
||
/**
|
||
* Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
|
||
* many. Each dropped class or signature is swept once (swept_at; null again when the record drops it anew),
|
||
* `perSweep` of them at most, each by its index (the claim's for a class, the signature index for a poster's
|
||
* rows, the ladder's for a tool's and group's) in rounds of `limit`. The prober runs this beside the lanes, not
|
||
* in the claim (LOG.md (133)); the edge drops new releases of dropped signatures and classes itself, so one
|
||
* sweep is all a drop needs.
|
||
*/
|
||
async discard(limit = 10_000, perSweep = 20): Promise<number> {
|
||
const { samplePercent } = await RuleSets.current()
|
||
let discarded = 0
|
||
const { rows: classes } = await db.rawQuery(
|
||
`SELECT tool, group_id, klass FROM pending_classes WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, klass LIMIT ?`,
|
||
[perSweep]
|
||
)
|
||
for (const c of classes) {
|
||
let rows: any[]
|
||
do {
|
||
;({ rows } = await db.rawQuery(
|
||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? LIMIT ?`,
|
||
[PENDING.waiting, c.tool, c.group_id, c.klass, limit]
|
||
))
|
||
discarded += await this.discardKeys(rows, samplePercent)
|
||
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
|
||
await db.rawQuery(
|
||
`UPDATE pending_classes SET swept_at = now() WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||
[c.tool, c.group_id, c.klass]
|
||
)
|
||
}
|
||
const { rows: signatures } = await db.rawQuery(
|
||
`SELECT tool, group_id, poster FROM pending_signatures WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, poster LIMIT ?`,
|
||
[perSweep]
|
||
)
|
||
for (const s of signatures) {
|
||
let rows: any[]
|
||
do {
|
||
// The poster's rows by the signature index (on the poster's md5: a poster can be longer than a btree
|
||
// key), a tool's and group's by the ladder's: never a test on the rows, which was a scan of the whole
|
||
// table per signature on the HDD (LOG.md (134)).
|
||
;({ rows } =
|
||
s.poster === ''
|
||
? await db.rawQuery(
|
||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||
WHERE status = ? AND group_id = ? AND tool = ? LIMIT ?`,
|
||
[PENDING.waiting, s.group_id, s.tool, limit]
|
||
)
|
||
: await db.rawQuery(
|
||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||
WHERE status = ? AND group_id = ? AND tool = ? AND md5(poster) = md5(?::text) AND poster = ? LIMIT ?`,
|
||
[PENDING.waiting, s.group_id, s.tool, s.poster, s.poster, limit]
|
||
))
|
||
discarded += await this.discardKeys(rows, samplePercent)
|
||
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
|
||
await db.rawQuery(
|
||
`UPDATE pending_signatures SET swept_at = now() WHERE tool = ? AND group_id = ? AND poster = ?`,
|
||
[s.tool, s.group_id, s.poster]
|
||
)
|
||
}
|
||
return discarded
|
||
}
|
||
|
||
/** The rows discarded (status 2), the explored sample excepted, their classes' waiting counts lowered; how many. */
|
||
private async discardKeys(rows: any[], samplePercent: number): Promise<number> {
|
||
const keys = rows
|
||
.map((r: any) => r.release_key as string)
|
||
.filter((key: string) => !explored(key, samplePercent))
|
||
.sort()
|
||
if (keys.length === 0) return 0
|
||
const { rows: gone } = await db.rawQuery(
|
||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
|
||
RETURNING tool, group_id, klass`,
|
||
[PENDING.discarded, PENDING.waiting, keys]
|
||
)
|
||
const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
|
||
for (const g of gone) {
|
||
const key = `${g.tool}\t${g.group_id}\t${g.klass}`
|
||
const entry = counts.get(key) ?? {
|
||
tool: g.tool,
|
||
group_id: Number(g.group_id),
|
||
klass: Number(g.klass),
|
||
n: 0,
|
||
}
|
||
entry.n++
|
||
counts.set(key, entry)
|
||
}
|
||
if (counts.size > 0) {
|
||
await db.rawQuery(
|
||
`UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
|
||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
|
||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||
[JSON.stringify(lockOrder(counts.values(), (c) => `${c.tool}\t${c.group_id}\t${c.klass}`))]
|
||
)
|
||
}
|
||
return gone.length
|
||
}
|
||
}
|
||
|
||
export default new PendingLadder()
|