Daan MeijerandClaude Fable 5.1
ae735361a0
edge-bench --group and replay:corpus take --to <article> next to --from, so a replay covers one window of a group's history while the corpus holds others (the 2016 ranges next to 2021's and 2025's). deploy/replay-window.sh runs a window on homelab: every group whose window ranges are all stored is replayed once into usenet_replay through the voyager-3 app image, 4 groups at a time, looping while the corpus fill completes the rest. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
70 lines
2.9 KiB
Bash
Executable File
70 lines
2.9 KiB
Bash
Executable File
#!/bin/bash
|
||
# A corpus window replayed on homelab (REBUILD.md, "The replay harness"): every group whose window ranges (the
|
||
# ranges marked `capture` at PRIORITY in the catalog, 0 for 2016) are all stored in the corpus is replayed once
|
||
# into the scratch catalog usenet_replay through the voyager-3 app image, PARALLEL groups at a time, the
|
||
# window's bounds passed as --from/--to (the corpus holds other windows of the same group). The group's log
|
||
# ends in .cache/replay/<group>.log, a .done marker next to it (.failed when the replay failed: remove it to
|
||
# retry). Loops until every group of the window is done (the corpus fill runs meanwhile), so run it detached
|
||
# from the deployment root:
|
||
# setsid nohup deploy/replay-window.sh > .cache/replay/driver.log 2>&1 &
|
||
# No new group starts while the nightly pause is on (.cache/paused); the running ones finish.
|
||
set -u
|
||
ROOT=${ROOT:-/mnt/bulk/usenet-rebuild}
|
||
PRIORITY=${PRIORITY:-0}
|
||
PARALLEL=${PARALLEL:-4}
|
||
MEMORY=${MEMORY:-12g}
|
||
IMAGE=${IMAGE:-usenet-voyager3:latest}
|
||
OUT=$ROOT/.cache/replay
|
||
mkdir -p "$OUT"
|
||
cd "$ROOT" || exit 1
|
||
|
||
say() { echo "$(date -Is) $*"; }
|
||
catalog() { docker compose exec -T postgres psql -U rebuild -d usenet_rebuild -X -q -At -F ' ' -c "$1"; }
|
||
|
||
# The window's groups: name, first article, last article, whether every range is stored; smallest first.
|
||
window() {
|
||
catalog "SELECT g.name, min(r.start_id), max(r.end_id), (count(*) = count(r.stored_at))::int
|
||
FROM ranges r JOIN groups g ON g.id = r.group_id
|
||
WHERE r.capture AND r.priority = $PRIORITY
|
||
GROUP BY g.name ORDER BY sum(r.end_id - r.start_id + 1)"
|
||
}
|
||
|
||
replay() {
|
||
local group=$1 from=$2 to=$3
|
||
say "$group: replaying $from–$to"
|
||
if docker run --rm --name "replay-${group//./-}" --network usenet-rebuild --env-file .env \
|
||
-e NODE_ENV=production -e DB_HOST=postgres -e DB_PORT=5432 -e DB_DATABASE=usenet_replay -e CORPUS_DIR=/corpus \
|
||
-v "$ROOT/corpus:/corpus:ro" --memory "$MEMORY" "$IMAGE" \
|
||
node ace replay:corpus --group "$group" --from "$from" --to "$to" > "$OUT/$group.log" 2>&1; then
|
||
touch "$OUT/$group.done"
|
||
say "$group: done ($(grep -o 'Releases [0-9,]*' "$OUT/$group.log" | tail -1))"
|
||
else
|
||
touch "$OUT/$group.failed"
|
||
say "$group: FAILED (see $OUT/$group.log)"
|
||
fi
|
||
rm -f "$OUT/$group.running"
|
||
}
|
||
|
||
while true; do
|
||
todo=0
|
||
while read -r group from to complete; do
|
||
[ -n "$group" ] || continue
|
||
[ -e "$OUT/$group.done" ] && continue
|
||
[ -e "$OUT/$group.failed" ] && continue
|
||
todo=$((todo + 1))
|
||
[ "$complete" = 1 ] || continue
|
||
[ -e "$OUT/$group.running" ] && continue
|
||
[ -e "$ROOT/.cache/paused" ] && continue
|
||
while [ "$(find "$OUT" -name '*.running' | wc -l)" -ge "$PARALLEL" ]; do sleep 30; done
|
||
touch "$OUT/$group.running"
|
||
replay "$group" "$from" "$to" &
|
||
sleep 5
|
||
done <<< "$(window)"
|
||
if [ "$todo" = 0 ]; then
|
||
wait
|
||
say "every group of the window at priority $PRIORITY is replayed"
|
||
exit 0
|
||
fi
|
||
sleep 300
|
||
done
|