Daan MeijerandClaude Fable 5.1
8f5f83b4ea
deploy.sh and the nightly resume leave the edge stopped while .cache/no-edge exists (range by range it would split releases at every edge with no merger left). deploy/replay-window.sh takes DB and OUT and passes REPLAY_INTO_CATALOG=1, which lets replay:corpus write a database not named *_replay or *_scratch (never the test one). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
397 lines
16 KiB
TypeScript
397 lines
16 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||
import { spawn } from 'node:child_process'
|
||
import { access, readdir, readFile, writeFile } from 'node:fs/promises'
|
||
import { tmpdir } from 'node:os'
|
||
import { join, resolve } from 'node:path'
|
||
import db from '@adonisjs/lucid/services/db'
|
||
import coordinatorConfig from '#config/coordinator'
|
||
import BatchStore from '#services/batch_store'
|
||
import Leases from '#services/leases'
|
||
import RuleSets from '#services/rule_sets'
|
||
import { fromWire, type WireBatch } from '#lib/edge/index'
|
||
|
||
/**
|
||
* Only a database named like one of these is written: the replay leases and finishes ranges and fills the catalog.
|
||
* Not the test database: the rows left behind failed 27 functional tests (LOG.md 2026-10-26 (111)). With
|
||
* REPLAY_INTO_CATALOG=1 any database but the test one: voyager 3 deployed, history replayed from the corpus into
|
||
* the catalog itself (deploy/replay-window.sh, LOG.md (116)).
|
||
*/
|
||
const SCRATCH_DB = /_(replay|scratch)$/
|
||
const TEST_DB = /_test$/
|
||
const NODE_NAME = 'replay'
|
||
|
||
interface StoredRange {
|
||
id: number
|
||
start: number
|
||
end: number
|
||
}
|
||
|
||
/**
|
||
* The replay harness (REBUILD.md, "Voyager 3", 8): a group's stored corpus ranges in article order through one
|
||
* window in the Rust edge (`edge-bench --group`, the window carried from range to range) and the batch store into a
|
||
* scratch catalog, as the worker would store them: each range leased to a node named "replay", its batches stored
|
||
* at its resume point, the range finished when its last batch is in. Then a report: releases finished, complete,
|
||
* partial and provisional, and the catalog's bytes per header (the NVMe tables apart from those on CHUNK_TABLESPACE).
|
||
*
|
||
* node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--from A] [--to B] [--limit N] [--batch-chunks 5]
|
||
*
|
||
* Refuses a database not named *_replay or *_scratch. The rules are the catalog's current rule set
|
||
* (rule_sets, the groups' priorities): copy `groups` and `rule_sets` from the catalog for a faithful replay.
|
||
*/
|
||
export default class ReplayCorpus extends BaseCommand {
|
||
static commandName = 'replay:corpus'
|
||
static description =
|
||
"Replays a group's corpus ranges through the Rust edge and the batch store into a scratch catalog"
|
||
static options: CommandOptions = { startApp: true }
|
||
|
||
@flags.string({ description: 'The group (its directory under the corpus)', required: true })
|
||
declare group: string
|
||
|
||
@flags.string({ description: 'The corpus directory (default: CORPUS_DIR)' })
|
||
declare dir?: string
|
||
|
||
@flags.number({ description: 'Replay this many ranges at most (default: all stored)' })
|
||
declare limit?: number
|
||
|
||
@flags.number({ description: 'Skip ranges ending before this article (continue a replay)' })
|
||
declare from?: number
|
||
|
||
@flags.number({
|
||
description: "Skip ranges starting after this article (one window of the group's history)",
|
||
})
|
||
declare to?: number
|
||
|
||
@flags.number({
|
||
description: 'Chunks of 100k articles per batch, as the worker posts them',
|
||
default: 5,
|
||
})
|
||
declare batchChunks: number
|
||
|
||
@flags.string({
|
||
description:
|
||
'The edge-bench binary (default: EDGE_BENCH, edge/target/release/edge-bench, /usr/local/bin/edge-bench)',
|
||
})
|
||
declare bench?: string
|
||
|
||
@flags.boolean({
|
||
description:
|
||
'Leave the window open at the end (the last ranges provisional), to continue later',
|
||
})
|
||
declare noFinish: boolean
|
||
|
||
async run() {
|
||
const database = db.connection().getReadClient().client.config.connection.database as string
|
||
const intoCatalog = process.env.REPLAY_INTO_CATALOG === '1' && !TEST_DB.test(database)
|
||
if (!SCRATCH_DB.test(database) && !intoCatalog) {
|
||
this.logger.error(
|
||
`The replay writes the catalog: it only runs against a database named *_replay or *_scratch (this is "${database}"; REPLAY_INTO_CATALOG=1 allows it)`
|
||
)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const dir = this.dir ?? coordinatorConfig.corpus.dir
|
||
if (!dir) {
|
||
this.logger.error('No corpus directory: --dir or CORPUS_DIR')
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const bench = await this.benchBinary()
|
||
if (!bench) {
|
||
this.logger.error(
|
||
'No edge-bench binary found: build it (cargo build --release --manifest-path edge/Cargo.toml) or set EDGE_BENCH'
|
||
)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const groupDir = join(dir, this.group)
|
||
const stored = await this.storedRanges(groupDir)
|
||
if (stored.length === 0) {
|
||
this.logger.error(`${groupDir}: no complete range`)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
const nodeId = await this.nodeId()
|
||
const groupId = await this.groupId()
|
||
const ranges = await this.leaseRanges(groupId, nodeId, stored)
|
||
const rules = await RuleSets.current()
|
||
const rulesPath = join(tmpdir(), `replay-rules-${process.pid}.json`)
|
||
await writeFile(rulesPath, JSON.stringify(rules))
|
||
this.logger.info(
|
||
`${this.group}: ${ranges.length} ranges (${ranges[0].start}–${ranges[ranges.length - 1].end}) through ${bench}, rules v${rules.version}, into ${database}.`
|
||
)
|
||
|
||
// edge-bench walks the group directory in article order itself; --from, --to and --limit cut its list as ours.
|
||
const args = [
|
||
'--group',
|
||
groupDir,
|
||
'--rules',
|
||
rulesPath,
|
||
'--batch-chunks',
|
||
String(this.batchChunks),
|
||
'--wire',
|
||
'-',
|
||
]
|
||
if (this.from) args.push('--from', String(this.from))
|
||
if (this.to) args.push('--to', String(this.to))
|
||
if (this.limit) args.push('--limit', String(this.limit))
|
||
if (this.noFinish) args.push('--no-finish')
|
||
const started = Date.now()
|
||
const child = spawn(bench, args, { stdio: ['ignore', 'pipe', 'inherit'] })
|
||
const totals = {
|
||
batches: 0,
|
||
headers: 0,
|
||
releases: 0,
|
||
newReleases: 0,
|
||
finalized: 0,
|
||
candidates: 0,
|
||
storeMs: 0,
|
||
}
|
||
let current: StoredRange | null = null
|
||
const storeLine = async (line: string) => {
|
||
const batch = fromWire(JSON.parse(line) as WireBatch)
|
||
if (!current || batch.from < current.start || batch.from > current.end) {
|
||
current = ranges.find((r) => batch.from >= r.start && batch.from <= r.end) ?? null
|
||
if (!current) throw new Error(`Batch at ${batch.from} is in no leased range`)
|
||
}
|
||
const at = Date.now()
|
||
const result = await BatchStore.store(current.id, nodeId, batch)
|
||
totals.storeMs += Date.now() - at
|
||
totals.batches++
|
||
totals.headers += batch.headers
|
||
totals.releases += result.releases
|
||
totals.newReleases += result.newReleases
|
||
totals.finalized += result.finalized
|
||
totals.candidates += result.candidates
|
||
if (batch.next > current.end) {
|
||
await Leases.finish(current.id, nodeId)
|
||
const seconds = (Date.now() - started) / 1000
|
||
this.logger.info(
|
||
` range ${current.start}–${current.end} done: ${totals.headers.toLocaleString('en')} headers so far in ${seconds.toFixed(0)} s ` +
|
||
`(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} releases stored (${totals.finalized} finished later), store ${(totals.storeMs / 1000).toFixed(0)} s`
|
||
)
|
||
}
|
||
}
|
||
// Lines pulled as the store keeps up: the child's stdout only moves when a chunk is read, so batches don't pile up.
|
||
let rest = ''
|
||
let failure: Error | null = null
|
||
try {
|
||
for await (const chunk of child.stdout as AsyncIterable<Buffer>) {
|
||
rest += chunk.toString('utf8')
|
||
let nl: number
|
||
while ((nl = rest.indexOf('\n')) >= 0) {
|
||
const line = rest.slice(0, nl)
|
||
rest = rest.slice(nl + 1)
|
||
if (line.length > 0) await storeLine(line)
|
||
}
|
||
}
|
||
if (rest.trim().length > 0) await storeLine(rest)
|
||
} catch (error: any) {
|
||
failure = error
|
||
child.kill('SIGTERM')
|
||
}
|
||
const code = await new Promise<number | null>((done) =>
|
||
child.exitCode !== null ? done(child.exitCode) : child.on('exit', done)
|
||
)
|
||
if (failure) {
|
||
this.logger.error(`Replay failed: ${failure.message}`)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
if (code !== 0) {
|
||
this.logger.error(`edge-bench exited with ${code}`)
|
||
this.exitCode = 1
|
||
return
|
||
}
|
||
await this.report(groupId, ranges, Date.now() - started)
|
||
}
|
||
|
||
private async benchBinary(): Promise<string | null> {
|
||
const candidates = [
|
||
this.bench,
|
||
process.env.EDGE_BENCH,
|
||
resolve('edge/target/release/edge-bench'),
|
||
'/usr/local/bin/edge-bench',
|
||
].filter((p): p is string => !!p)
|
||
for (const path of candidates) {
|
||
if (
|
||
await access(path).then(
|
||
() => true,
|
||
() => false
|
||
)
|
||
)
|
||
return path
|
||
}
|
||
return null
|
||
}
|
||
|
||
/** The group directory's complete ranges (a range.json), in article order, cut by --from, --to and --limit. */
|
||
private async storedRanges(groupDir: string): Promise<{ start: number; end: number }[]> {
|
||
const names = await readdir(groupDir).catch(() => [] as string[])
|
||
const ranges: { start: number; end: number }[] = []
|
||
for (const name of names) {
|
||
const m = /^(\d+)-(\d+)$/.exec(name)
|
||
if (!m) continue
|
||
const json = await readFile(join(groupDir, name, 'range.json'), 'utf8').catch(() => null)
|
||
if (!json) continue
|
||
ranges.push({ start: Number(m[1]), end: Number(m[2]) })
|
||
}
|
||
ranges.sort((a, b) => a.start - b.start)
|
||
let chosen = this.from ? ranges.filter((r) => r.end >= this.from!) : ranges
|
||
if (this.to) chosen = chosen.filter((r) => r.start <= this.to!)
|
||
if (this.limit) chosen = chosen.slice(0, this.limit)
|
||
return chosen
|
||
}
|
||
|
||
private async nodeId(): Promise<number> {
|
||
const row = await db.from('nodes').where('name', NODE_NAME).first()
|
||
if (row) return Number(row.id)
|
||
const [inserted] = await db.table('nodes').insert({ name: NODE_NAME }).returning('id')
|
||
return Number(inserted.id)
|
||
}
|
||
|
||
/** The group's row (inserted after the last priority when the scratch catalog doesn't have it). */
|
||
private async groupId(): Promise<number> {
|
||
const row = await db.from('groups').where('name', this.group).first()
|
||
if (row) return Number(row.id)
|
||
const { rows } = await db.rawQuery(`SELECT coalesce(max(priority), 0) + 1 AS next FROM groups`)
|
||
const [inserted] = await db
|
||
.table('groups')
|
||
.insert({ name: this.group, priority: Number(rows[0].next), active: true })
|
||
.returning('id')
|
||
this.logger.info(`Group ${this.group} added with priority ${rows[0].next}.`)
|
||
return Number(inserted.id)
|
||
}
|
||
|
||
/** The ranges as leased to the replay node, from their start (an earlier replay's rows are taken over). */
|
||
private async leaseRanges(
|
||
groupId: number,
|
||
nodeId: number,
|
||
stored: { start: number; end: number }[]
|
||
): Promise<StoredRange[]> {
|
||
const out: StoredRange[] = []
|
||
const now = new Date()
|
||
for (const r of stored) {
|
||
const lease = {
|
||
status: 'leased',
|
||
node_id: nodeId,
|
||
next_id: r.start,
|
||
headers: 0,
|
||
posted_low: null,
|
||
posted_high: null,
|
||
leased_at: now,
|
||
heartbeat_at: now,
|
||
done_at: null,
|
||
error: null,
|
||
capture: true,
|
||
}
|
||
const existing = await db
|
||
.from('ranges')
|
||
.where('group_id', groupId)
|
||
.where('start_id', r.start)
|
||
.first()
|
||
if (existing) {
|
||
await db
|
||
.from('ranges')
|
||
.where('id', existing.id)
|
||
.update({ ...lease, end_id: r.end, attempts: Number(existing.attempts) + 1 })
|
||
out.push({ id: Number(existing.id), start: r.start, end: r.end })
|
||
continue
|
||
}
|
||
const [inserted] = await db
|
||
.table('ranges')
|
||
.insert({
|
||
...lease,
|
||
group_id: groupId,
|
||
start_id: r.start,
|
||
end_id: r.end,
|
||
priority: 0,
|
||
attempts: 1,
|
||
})
|
||
.returning('id')
|
||
out.push({ id: Number(inserted.id), start: r.start, end: r.end })
|
||
}
|
||
return out
|
||
}
|
||
|
||
/** What the replay left in the catalog (this group's releases; the sizes are the whole catalog's), per article. */
|
||
private async report(groupId: number, ranges: StoredRange[], elapsedMs: number) {
|
||
const ids = ranges.map((r) => r.id)
|
||
const { rows: counts } = await db.rawQuery(
|
||
`SELECT count(*)::bigint AS releases,
|
||
count(settled_at)::bigint AS finished,
|
||
count(*) FILTER (WHERE settled_at IS NULL)::bigint AS provisional,
|
||
count(*) FILTER (WHERE edge)::bigint AS edges,
|
||
count(*) FILTER (WHERE complete)::bigint AS complete,
|
||
count(*) FILTER (WHERE complete = false)::bigint AS incomplete,
|
||
count(*) FILTER (WHERE complete IS NULL)::bigint AS undecided,
|
||
count(*) FILTER (WHERE readable)::bigint AS readable,
|
||
count(*) FILTER (WHERE has_par2)::bigint AS with_par2,
|
||
coalesce(sum(files), 0)::bigint AS files,
|
||
coalesce(sum(segments), 0)::bigint AS segments,
|
||
coalesce(sum(bytes), 0)::bigint AS bytes
|
||
FROM releases WHERE group_id = ?`,
|
||
[groupId]
|
||
)
|
||
const { rows: sources } = await db.rawQuery(
|
||
`SELECT source, count(*)::bigint AS n FROM releases WHERE group_id = ? GROUP BY source ORDER BY source`,
|
||
[groupId]
|
||
)
|
||
const { rows: pending } = await db.rawQuery(
|
||
`SELECT count(*)::bigint AS n, coalesce(sum(segments), 0)::bigint AS segments, count(*) FILTER (WHERE jsonb_array_length(candidates) = 0)::bigint AS unprobable
|
||
FROM pending_releases WHERE group_id = ?`,
|
||
[groupId]
|
||
)
|
||
const { rows: headerRows } = await db.rawQuery(
|
||
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM ranges WHERE id = ANY(?::bigint[])`,
|
||
[ids]
|
||
)
|
||
const headers = Number(headerRows[0].headers)
|
||
const { rows: sizes } = await db.rawQuery(
|
||
`SELECT CASE WHEN c.relname LIKE 'release_chunks%' THEN 'release_chunks' ELSE c.relname END AS name,
|
||
sum(pg_total_relation_size(c.oid))::bigint AS bytes
|
||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||
WHERE n.nspname = 'public' AND c.relkind = 'r'
|
||
AND (c.relname IN ('releases', 'release_texts', 'posters', 'group_sets', 'probe_candidates', 'rule_counts', 'rule_samples', 'blocks', 'pending_releases')
|
||
OR c.relname LIKE 'release_chunks%')
|
||
GROUP BY 1 ORDER BY 2 DESC`
|
||
)
|
||
// On CHUNK_TABLESPACE (the HDD) in the catalog: the chunks (migration 3200), inspections and candidates (6000).
|
||
const hdd = new Set([
|
||
'release_chunks',
|
||
'probe_candidates',
|
||
'release_inspections',
|
||
'pending_releases',
|
||
])
|
||
let nvme = 0
|
||
let other = 0
|
||
const c = counts[0]
|
||
const fmt = (n: unknown) => Number(n).toLocaleString('en')
|
||
this.logger.info(
|
||
`Replay of ${ranges.length} ranges: ${fmt(headers)} headers in ${(elapsedMs / 1000).toFixed(0)} s (${fmt(Math.round(headers / Math.max(1, elapsedMs / 1000)))}/s).`
|
||
)
|
||
this.logger.info(
|
||
`Releases ${fmt(c.releases)}: finished ${fmt(c.finished)}, provisional ${fmt(c.provisional)}, edges ${fmt(c.edges)}; ` +
|
||
`complete ${fmt(c.complete)}, incomplete ${fmt(c.incomplete)}, undecided ${fmt(c.undecided)}; readable ${fmt(c.readable)}, with par2 ${fmt(c.with_par2)}; ` +
|
||
`${fmt(c.files)} files, ${fmt(c.segments)} segments (${((Number(c.segments) / Math.max(1, headers)) * 100).toFixed(1)}% of the headers), ${(Number(c.bytes) / 1e12).toFixed(2)} TB.`
|
||
)
|
||
this.logger.info(
|
||
`By source: ${sources.map((s: any) => `${s.source}: ${fmt(s.n)}`).join(', ')}.`
|
||
)
|
||
this.logger.info(
|
||
`Pending (unnamed, outside the catalog): ${fmt(pending[0].n)} releases, ${fmt(pending[0].segments)} segments, ${fmt(pending[0].unprobable)} without a probe target.`
|
||
)
|
||
for (const s of sizes) {
|
||
const bytes = Number(s.bytes)
|
||
if (hdd.has(s.name)) other += bytes
|
||
else nvme += bytes
|
||
this.logger.info(
|
||
` ${s.name}: ${(bytes / 1e6).toFixed(1)} MB, ${(bytes / Math.max(1, headers)).toFixed(2)} B/header`
|
||
)
|
||
}
|
||
this.logger.info(
|
||
`NVMe tables: ${(nvme / 1e6).toFixed(1)} MB, ${(nvme / Math.max(1, headers)).toFixed(2)} B/header; HDD tables (CHUNK_TABLESPACE): ${(other / 1e6).toFixed(1)} MB, ${(other / Math.max(1, headers)).toFixed(2)} B/header (the whole catalog's sizes over this group's headers: only the first group's are its own).`
|
||
)
|
||
}
|
||
}
|