Replay harness: a --to bound and the homelab window driver

edge-bench --group and replay:corpus take --to <article> next to --from, so a
replay covers one window of a group's history while the corpus holds others
(the 2016 ranges next to 2021's and 2025's). deploy/replay-window.sh runs a
window on homelab: every group whose window ranges are all stored is replayed
once into usenet_replay through the voyager-3 app image, 4 groups at a time,
looping while the corpus fill completes the rest.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

deploy/replay-window.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# A corpus window replayed on homelab (REBUILD.md, "The replay harness"): every group whose window ranges (the
+# ranges marked `capture` at PRIORITY in the catalog, 0 for 2016) are all stored in the corpus is replayed once
+# into the scratch catalog usenet_replay through the voyager-3 app image, PARALLEL groups at a time, the
+# window's bounds passed as --from/--to (the corpus holds other windows of the same group). The group's log
+# ends in .cache/replay/<group>.log, a .done marker next to it (.failed when the replay failed: remove it to
+# retry). Loops until every group of the window is done (the corpus fill runs meanwhile), so run it detached
+# from the deployment root:
+#   setsid nohup deploy/replay-window.sh > .cache/replay/driver.log 2>&1 &
+# No new group starts while the nightly pause is on (.cache/paused); the running ones finish.
+set -u
+ROOT=${ROOT:-/mnt/bulk/usenet-rebuild}
+PRIORITY=${PRIORITY:-0}
+PARALLEL=${PARALLEL:-4}
+MEMORY=${MEMORY:-12g}
+IMAGE=${IMAGE:-usenet-voyager3:latest}
+OUT=$ROOT/.cache/replay
+mkdir -p "$OUT"
+cd "$ROOT" || exit 1
+
+say() { echo "$(date -Is) $*"; }
+catalog() { docker compose exec -T postgres psql -U rebuild -d usenet_rebuild -X -q -At -F ' ' -c "$1"; }
+
+# The window's groups: name, first article, last article, whether every range is stored; smallest first.
+window() {
+  catalog "SELECT g.name, min(r.start_id), max(r.end_id), (count(*) = count(r.stored_at))::int
+           FROM ranges r JOIN groups g ON g.id = r.group_id
+           WHERE r.capture AND r.priority = $PRIORITY
+           GROUP BY g.name ORDER BY sum(r.end_id - r.start_id + 1)"
+}
+
+replay() {
+  local group=$1 from=$2 to=$3
+  say "$group: replaying $from–$to"
+  if docker run --rm --name "replay-${group//./-}" --network usenet-rebuild --env-file .env \
+      -e NODE_ENV=production -e DB_HOST=postgres -e DB_PORT=5432 -e DB_DATABASE=usenet_replay -e CORPUS_DIR=/corpus \
+      -v "$ROOT/corpus:/corpus:ro" --memory "$MEMORY" "$IMAGE" \
+      node ace replay:corpus --group "$group" --from "$from" --to "$to" > "$OUT/$group.log" 2>&1; then
+    touch "$OUT/$group.done"
+    say "$group: done ($(grep -o 'Releases [0-9,]*' "$OUT/$group.log" | tail -1))"
+  else
+    touch "$OUT/$group.failed"
+    say "$group: FAILED (see $OUT/$group.log)"
+  fi
+  rm -f "$OUT/$group.running"
+}
+
+while true; do
+  todo=0
+  while read -r group from to complete; do
+    [ -n "$group" ] || continue
+    [ -e "$OUT/$group.done" ] && continue
+    [ -e "$OUT/$group.failed" ] && continue
+    todo=$((todo + 1))
+    [ "$complete" = 1 ] || continue
+    [ -e "$OUT/$group.running" ] && continue
+    [ -e "$ROOT/.cache/paused" ] && continue
+    while [ "$(find "$OUT" -name '*.running' | wc -l)" -ge "$PARALLEL" ]; do sleep 30; done
+    touch "$OUT/$group.running"
+    replay "$group" "$from" "$to" &
+    sleep 5
+  done <<< "$(window)"
+  if [ "$todo" = 0 ]; then
+    wait
+    say "every group of the window at priority $PRIORITY is replayed"
+    exit 0
+  fi
+  sleep 300
+done