Replay harness: a --to bound and the homelab window driver
edge-bench --group and replay:corpus take --to <article> next to --from, so a replay covers one window of a group's history while the corpus holds others (the 2016 ranges next to 2021's and 2025's). deploy/replay-window.sh runs a window on homelab: every group whose window ranges are all stored is replayed once into usenet_replay through the voyager-3 app image, 4 groups at a time, looping while the corpus fill completes the rest. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
72e973d006
commit
ae735361a0
5 files changed
+99
-12
No files matched your search
@@ -31,7 +31,7 @@ interface StoredRange {
|
||||
* at its resume point, the range finished when its last batch is in. Then a report: releases finished, complete,
|
||||
* partial and provisional, and the catalog's bytes per header (the NVMe tables apart from those on CHUNK_TABLESPACE).
|
||||
*
|
||||
* node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--limit N] [--batch-chunks 5]
|
||||
* node ace replay:corpus --group alt.binaries.ath [--dir CORPUS_DIR] [--from A] [--to B] [--limit N] [--batch-chunks 5]
|
||||
*
|
||||
* Refuses a database not named *_replay or *_scratch. The rules are the catalog's current rule set
|
||||
* (rule_sets, the groups' priorities): copy `groups` and `rule_sets` from the catalog for a faithful replay.
|
||||
@@ -54,6 +54,11 @@ export default class ReplayCorpus extends BaseCommand {
|
||||
@flags.number({ description: 'Skip ranges ending before this article (continue a replay)' })
|
||||
declare from?: number
|
||||
|
||||
@flags.number({
|
||||
description: "Skip ranges starting after this article (one window of the group's history)",
|
||||
})
|
||||
declare to?: number
|
||||
|
||||
@flags.number({
|
||||
description: 'Chunks of 100k articles per batch, as the worker posts them',
|
||||
default: 5,
|
||||
@@ -112,7 +117,7 @@ export default class ReplayCorpus extends BaseCommand {
|
||||
`${this.group}: ${ranges.length} ranges (${ranges[0].start}–${ranges[ranges.length - 1].end}) through ${bench}, rules v${rules.version}, into ${database}.`
|
||||
)
|
||||
|
||||
// edge-bench walks the group directory in article order itself; --from and --limit cut its list as ours.
|
||||
// edge-bench walks the group directory in article order itself; --from, --to and --limit cut its list as ours.
|
||||
const args = [
|
||||
'--group',
|
||||
groupDir,
|
||||
@@ -124,6 +129,7 @@ export default class ReplayCorpus extends BaseCommand {
|
||||
'-',
|
||||
]
|
||||
if (this.from) args.push('--from', String(this.from))
|
||||
if (this.to) args.push('--to', String(this.to))
|
||||
if (this.limit) args.push('--limit', String(this.limit))
|
||||
if (this.noFinish) args.push('--no-finish')
|
||||
const started = Date.now()
|
||||
@@ -215,7 +221,7 @@ export default class ReplayCorpus extends BaseCommand {
|
||||
return null
|
||||
}
|
||||
|
||||
/** The group directory's complete ranges (a range.json), in article order, cut by --from and --limit. */
|
||||
/** The group directory's complete ranges (a range.json), in article order, cut by --from, --to and --limit. */
|
||||
private async storedRanges(groupDir: string): Promise<{ start: number; end: number }[]> {
|
||||
const names = await readdir(groupDir).catch(() => [] as string[])
|
||||
const ranges: { start: number; end: number }[] = []
|
||||
@@ -228,6 +234,7 @@ export default class ReplayCorpus extends BaseCommand {
|
||||
}
|
||||
ranges.sort((a, b) => a.start - b.start)
|
||||
let chosen = this.from ? ranges.filter((r) => r.end >= this.from!) : ranges
|
||||
if (this.to) chosen = chosen.filter((r) => r.start <= this.to!)
|
||||
if (this.limit) chosen = chosen.slice(0, this.limit)
|
||||
return chosen
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user