Daan MeijerandClaude Fable 5.1
72e973d006
replay:corpus runs edge-bench --group over a group's complete corpus ranges in article order (one window, carried from range to range, finished after the last range; edge/src/lease.rs holds the lease JSON the worker and the bench share) and stores every batch through BatchStore as the worker would, then reports releases finished, provisional, complete and the bytes per header of the NVMe and HDD tables. Refuses a database not named *_replay or *_scratch. The app image carries edge-bench; compose gets a replay profile for homelab. Design amended (REBUILD.md 1, 2, 5): the window is carried by one worker a group, no block leases and no block merger; provisional writes stay for the lease ends. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
371 lines
15 KiB
TypeScript
371 lines
15 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||
import { spawn } from 'node:child_process'
|
||
import { access, readdir, readFile, writeFile } from 'node:fs/promises'
|
||
import { tmpdir } from 'node:os'
|
||
import { join, resolve } from 'node:path'
|
||
import db from '@adonisjs/lucid/services/db'
|
||
import coordinatorConfig from '#config/coordinator'
|
||
import BatchStore from '#services/batch_store'
|
||
import Leases from '#services/leases'
|
||
import RuleSets from '#services/rule_sets'
|
||
import { fromWire, type WireBatch } from '#lib/edge/index'
|
||
|
||
/**
|
||
* Only a database named like one of these is written: the replay leases and finishes ranges and fills the catalog.
|
||
* Not the test database: the rows left behind failed 27 functional tests (LOG.md 2026-10-26 (111)).
|
||
*/
|
||
const SCRATCH_DB = /_(replay|scratch)$/
|
||
const NODE_NAME = 'replay'
|
||
|
||
interface StoredRange {
|
||
id: number
|
||
start: number
|
||
end: number
|
||
}
|
||
|
||
/**
|
||
* The replay harness (REBUILD.md, "Voyager 3", 8): a group's stored corpus ranges in article order through one
|
||
* window in the Rust edge (`edge-bench --group`, the window carried from range to range) and the batch store into a
|
||
* scratch catalog, as the worker would store them: each range leased to a node named "replay", its batches stored
|
||
* at its resume point, the range finished when its last batch is in. Then a report: releases finished, complete,
|
||
* partial and provisional, and the catalog's bytes per header (the NVMe tables apart from those on CHUNK_TABLESPACE).
|
||
*
|
||
* node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--limit N] [--batch-chunks 5]
|
||
*
|
||
* Refuses a database not named *_replay or *_scratch. The rules are the catalog's current rule set
|
||
* (rule_sets, the groups' priorities): copy `groups` and `rule_sets` from the catalog for a faithful replay.
|
||
*/
|
||
export default class ReplayCorpus extends BaseCommand {
|
||
static commandName = 'replay:corpus'
|
||
static description =
|
||
"Replays a group's corpus ranges through the Rust edge and the batch store into a scratch catalog"
|
||
static options: CommandOptions = { startApp: true }
|
||
|
||
@flags.string({ description: 'The group (its directory under the corpus)', required: true })
|
||
declare group: string
|
||
|
||
@flags.string({ description: 'The corpus directory (default: CORPUS_DIR)' })
|
||
declare dir?: string
|
||
|
||
@flags.number({ description: 'Replay this many ranges at most (default: all stored)' })
|
||
declare limit?: number
|
||
|
||
@flags.number({ description: 'Skip ranges ending before this article (continue a replay)' })
|
||
declare from?: number
|
||
|
||
@flags.number({
|
||
description: 'Chunks of 100k articles per batch, as the worker posts them',
|
||
default: 5,
|
||
})
|
||
declare batchChunks: number
|
||
|
||
@flags.string({
|
||
description:
|
||
'The edge-bench binary (default: EDGE_BENCH, edge/target/release/edge-bench, /usr/local/bin/edge-bench)',
|
||
})
|
||
declare bench?: string
|
||
|
||
@flags.boolean({
|
||
description:
|
||
'Leave the window open at the end (the last ranges provisional), to continue later',
|
||
})
|
||
declare noFinish: boolean
|
||
|
||
async run() {
|
||
const database = db.connection().getReadClient().client.config.connection.database as string
|
||
if (!SCRATCH_DB.test(database)) {
|
||
this.logger.error(
|
||
`The replay writes the catalog: it only runs against a database named *_replay or *_scratch (this is "${database}")`
|
||
)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const dir = this.dir ?? coordinatorConfig.corpus.dir
|
||
if (!dir) {
|
||
this.logger.error('No corpus directory: --dir or CORPUS_DIR')
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const bench = await this.benchBinary()
|
||
if (!bench) {
|
||
this.logger.error(
|
||
'No edge-bench binary found: build it (cargo build --release --manifest-path edge/Cargo.toml) or set EDGE_BENCH'
|
||
)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const groupDir = join(dir, this.group)
|
||
const stored = await this.storedRanges(groupDir)
|
||
if (stored.length === 0) {
|
||
this.logger.error(`${groupDir}: no complete range`)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const nodeId = await this.nodeId()
|
||
const groupId = await this.groupId()
|
||
const ranges = await this.leaseRanges(groupId, nodeId, stored)
|
||
const rules = await RuleSets.current()
|
||
const rulesPath = join(tmpdir(), `replay-rules-${process.pid}.json`)
|
||
await writeFile(rulesPath, JSON.stringify(rules))
|
||
this.logger.info(
|
||
`${this.group}: ${ranges.length} ranges (${ranges[0].start}–${ranges[ranges.length - 1].end}) through ${bench}, rules v${rules.version}, into ${database}.`
|
||
)
|
||
|
||
// edge-bench walks the group directory in article order itself; --from and --limit cut its list as ours.
|
||
const args = [
|
||
'--group',
|
||
groupDir,
|
||
'--rules',
|
||
rulesPath,
|
||
'--batch-chunks',
|
||
String(this.batchChunks),
|
||
'--wire',
|
||
'-',
|
||
]
|
||
if (this.from) args.push('--from', String(this.from))
|
||
if (this.limit) args.push('--limit', String(this.limit))
|
||
if (this.noFinish) args.push('--no-finish')
|
||
const started = Date.now()
|
||
const child = spawn(bench, args, { stdio: ['ignore', 'pipe', 'inherit'] })
|
||
const totals = {
|
||
batches: 0,
|
||
headers: 0,
|
||
releases: 0,
|
||
newReleases: 0,
|
||
finalized: 0,
|
||
candidates: 0,
|
||
storeMs: 0,
|
||
}
|
||
let current: StoredRange | null = null
|
||
const storeLine = async (line: string) => {
|
||
const batch = fromWire(JSON.parse(line) as WireBatch)
|
||
if (!current || batch.from < current.start || batch.from > current.end) {
|
||
current = ranges.find((r) => batch.from >= r.start && batch.from <= r.end) ?? null
|
||
if (!current) throw new Error(`Batch at ${batch.from} is in no leased range`)
|
||
}
|
||
const at = Date.now()
|
||
const result = await BatchStore.store(current.id, nodeId, batch)
|
||
totals.storeMs += Date.now() - at
|
||
totals.batches++
|
||
totals.headers += batch.headers
|
||
totals.releases += result.releases
|
||
totals.newReleases += result.newReleases
|
||
totals.finalized += result.finalized
|
||
totals.candidates += result.candidates
|
||
if (batch.next > current.end) {
|
||
await Leases.finish(current.id, nodeId)
|
||
const seconds = (Date.now() - started) / 1000
|
||
this.logger.info(
|
||
` range ${current.start}–${current.end} done: ${totals.headers.toLocaleString('en')} headers so far in ${seconds.toFixed(0)} s ` +
|
||
`(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} releases stored (${totals.finalized} finished later), store ${(totals.storeMs / 1000).toFixed(0)} s`
|
||
)
|
||
}
|
||
}
|
||
// Lines pulled as the store keeps up: the child's stdout only moves when a chunk is read, so batches don't pile up.
|
||
let rest = ''
|
||
let failure: Error | null = null
|
||
try {
|
||
for await (const chunk of child.stdout as AsyncIterable<Buffer>) {
|
||
rest += chunk.toString('utf8')
|
||
let nl: number
|
||
while ((nl = rest.indexOf('\n')) >= 0) {
|
||
const line = rest.slice(0, nl)
|
||
rest = rest.slice(nl + 1)
|
||
if (line.length > 0) await storeLine(line)
|
||
}
|
||
}
|
||
if (rest.trim().length > 0) await storeLine(rest)
|
||
} catch (error: any) {
|
||
failure = error
|
||
child.kill('SIGTERM')
|
||
}
|
||
const code = await new Promise<number | null>((done) =>
|
||
child.exitCode !== null ? done(child.exitCode) : child.on('exit', done)
|
||
)
|
||
if (failure) {
|
||
this.logger.error(`Replay failed: ${failure.message}`)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
if (code !== 0) {
|
||
this.logger.error(`edge-bench exited with ${code}`)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
await this.report(ranges, Date.now() - started)
|
||
}
|
||
|
||
private async benchBinary(): Promise<string | null> {
|
||
const candidates = [
|
||
this.bench,
|
||
process.env.EDGE_BENCH,
|
||
resolve('edge/target/release/edge-bench'),
|
||
'/usr/local/bin/edge-bench',
|
||
].filter((p): p is string => !!p)
|
||
for (const path of candidates) {
|
||
if (
|
||
await access(path).then(
|
||
() => true,
|
||
() => false
|
||
)
|
||
)
|
||
return path
|
||
}
|
||
return null
|
||
}
|
||
|
||
/** The group directory's complete ranges (a range.json), in article order, cut by --from and --limit. */
|
||
private async storedRanges(groupDir: string): Promise<{ start: number; end: number }[]> {
|
||
const names = await readdir(groupDir).catch(() => [] as string[])
|
||
const ranges: { start: number; end: number }[] = []
|
||
for (const name of names) {
|
||
const m = /^(\d+)-(\d+)$/.exec(name)
|
||
if (!m) continue
|
||
const json = await readFile(join(groupDir, name, 'range.json'), 'utf8').catch(() => null)
|
||
if (!json) continue
|
||
ranges.push({ start: Number(m[1]), end: Number(m[2]) })
|
||
}
|
||
ranges.sort((a, b) => a.start - b.start)
|
||
let chosen = this.from ? ranges.filter((r) => r.end >= this.from!) : ranges
|
||
if (this.limit) chosen = chosen.slice(0, this.limit)
|
||
return chosen
|
||
}
|
||
|
||
private async nodeId(): Promise<number> {
|
||
const row = await db.from('nodes').where('name', NODE_NAME).first()
|
||
if (row) return Number(row.id)
|
||
const [inserted] = await db.table('nodes').insert({ name: NODE_NAME }).returning('id')
|
||
return Number(inserted.id)
|
||
}
|
||
|
||
/** The group's row (inserted after the last priority when the scratch catalog doesn't have it). */
|
||
private async groupId(): Promise<number> {
|
||
const row = await db.from('groups').where('name', this.group).first()
|
||
if (row) return Number(row.id)
|
||
const { rows } = await db.rawQuery(`SELECT coalesce(max(priority), 0) + 1 AS next FROM groups`)
|
||
const [inserted] = await db
|
||
.table('groups')
|
||
.insert({ name: this.group, priority: Number(rows[0].next), active: true })
|
||
.returning('id')
|
||
this.logger.info(`Group ${this.group} added with priority ${rows[0].next}.`)
|
||
return Number(inserted.id)
|
||
}
|
||
|
||
/** The ranges as leased to the replay node, from their start (an earlier replay's rows are taken over). */
|
||
private async leaseRanges(
|
||
groupId: number,
|
||
nodeId: number,
|
||
stored: { start: number; end: number }[]
|
||
): Promise<StoredRange[]> {
|
||
const out: StoredRange[] = []
|
||
const now = new Date()
|
||
for (const r of stored) {
|
||
const lease = {
|
||
status: 'leased',
|
||
node_id: nodeId,
|
||
next_id: r.start,
|
||
headers: 0,
|
||
posted_low: null,
|
||
posted_high: null,
|
||
leased_at: now,
|
||
heartbeat_at: now,
|
||
done_at: null,
|
||
error: null,
|
||
capture: true,
|
||
}
|
||
const existing = await db
|
||
.from('ranges')
|
||
.where('group_id', groupId)
|
||
.where('start_id', r.start)
|
||
.first()
|
||
if (existing) {
|
||
await db
|
||
.from('ranges')
|
||
.where('id', existing.id)
|
||
.update({ ...lease, end_id: r.end, attempts: Number(existing.attempts) + 1 })
|
||
out.push({ id: Number(existing.id), start: r.start, end: r.end })
|
||
continue
|
||
}
|
||
const [inserted] = await db
|
||
.table('ranges')
|
||
.insert({
|
||
...lease,
|
||
group_id: groupId,
|
||
start_id: r.start,
|
||
end_id: r.end,
|
||
priority: 0,
|
||
attempts: 1,
|
||
})
|
||
.returning('id')
|
||
out.push({ id: Number(inserted.id), start: r.start, end: r.end })
|
||
}
|
||
return out
|
||
}
|
||
|
||
/** What the replay left in the catalog, and its size per article. */
|
||
private async report(ranges: StoredRange[], elapsedMs: number) {
|
||
const ids = ranges.map((r) => r.id)
|
||
const { rows: counts } = await db.rawQuery(
|
||
`SELECT count(*)::bigint AS releases,
|
||
count(settled_at)::bigint AS finished,
|
||
count(*) FILTER (WHERE settled_at IS NULL)::bigint AS provisional,
|
||
count(*) FILTER (WHERE edge)::bigint AS edges,
|
||
count(*) FILTER (WHERE complete)::bigint AS complete,
|
||
count(*) FILTER (WHERE complete = false)::bigint AS incomplete,
|
||
count(*) FILTER (WHERE complete IS NULL)::bigint AS undecided,
|
||
count(*) FILTER (WHERE readable)::bigint AS readable,
|
||
count(*) FILTER (WHERE has_par2)::bigint AS with_par2,
|
||
coalesce(sum(files), 0)::bigint AS files,
|
||
coalesce(sum(segments), 0)::bigint AS segments,
|
||
coalesce(sum(bytes), 0)::bigint AS bytes
|
||
FROM releases`
|
||
)
|
||
const { rows: sources } = await db.rawQuery(
|
||
`SELECT source, count(*)::bigint AS n FROM releases GROUP BY source ORDER BY source`
|
||
)
|
||
const { rows: headerRows } = await db.rawQuery(
|
||
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM ranges WHERE id = ANY(?::bigint[])`,
|
||
[ids]
|
||
)
|
||
const headers = Number(headerRows[0].headers)
|
||
const { rows: sizes } = await db.rawQuery(
|
||
`SELECT CASE WHEN c.relname LIKE 'release_chunks%' THEN 'release_chunks' ELSE c.relname END AS name,
|
||
sum(pg_total_relation_size(c.oid))::bigint AS bytes
|
||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||
WHERE n.nspname = 'public' AND c.relkind = 'r'
|
||
AND (c.relname IN ('releases', 'release_texts', 'posters', 'group_sets', 'probe_candidates', 'rule_counts', 'rule_samples', 'blocks')
|
||
OR c.relname LIKE 'release_chunks%')
|
||
GROUP BY 1 ORDER BY 2 DESC`
|
||
)
|
||
// On CHUNK_TABLESPACE (the HDD) in the catalog: the chunks (migration 3200), inspections and candidates (6000).
|
||
const hdd = new Set(['release_chunks', 'probe_candidates', 'release_inspections'])
|
||
let nvme = 0
|
||
let other = 0
|
||
const c = counts[0]
|
||
const fmt = (n: unknown) => Number(n).toLocaleString('en')
|
||
this.logger.info(
|
||
`Replay of ${ranges.length} ranges: ${fmt(headers)} headers in ${(elapsedMs / 1000).toFixed(0)} s (${fmt(Math.round(headers / Math.max(1, elapsedMs / 1000)))}/s).`
|
||
)
|
||
this.logger.info(
|
||
`Releases ${fmt(c.releases)}: finished ${fmt(c.finished)}, provisional ${fmt(c.provisional)}, edges ${fmt(c.edges)}; ` +
|
||
`complete ${fmt(c.complete)}, incomplete ${fmt(c.incomplete)}, undecided ${fmt(c.undecided)}; readable ${fmt(c.readable)}, with par2 ${fmt(c.with_par2)}; ` +
|
||
`${fmt(c.files)} files, ${fmt(c.segments)} segments (${((Number(c.segments) / Math.max(1, headers)) * 100).toFixed(1)}% of the headers), ${(Number(c.bytes) / 1e12).toFixed(2)} TB.`
|
||
)
|
||
this.logger.info(
|
||
`By source: ${sources.map((s: any) => `${s.source}: ${fmt(s.n)}`).join(', ')}.`
|
||
)
|
||
for (const s of sizes) {
|
||
const bytes = Number(s.bytes)
|
||
if (hdd.has(s.name)) other += bytes
|
||
else nvme += bytes
|
||
this.logger.info(
|
||
` ${s.name}: ${(bytes / 1e6).toFixed(1)} MB, ${(bytes / Math.max(1, headers)).toFixed(2)} B/header`
|
||
)
|
||
}
|
||
this.logger.info(
|
||
`NVMe tables: ${(nvme / 1e6).toFixed(1)} MB, ${(nvme / Math.max(1, headers)).toFixed(2)} B/header; HDD tables (CHUNK_TABLESPACE): ${(other / 1e6).toFixed(1)} MB, ${(other / Math.max(1, headers)).toFixed(2)} B/header.`
|
||
)
|
||
}
|
||
}
|