#!/bin/bash
# A corpus window replayed on homelab (REBUILD.md, "The replay harness"): every group whose window ranges (the
# ranges marked `capture` at PRIORITY in the catalog, 0 for 2016) are all stored in the corpus is replayed once
# into the scratch catalog usenet_replay through the voyager-3 app image, PARALLEL groups at a time, the
# window's bounds passed as --from/--to (the corpus holds other windows of the same group). The group's log
# ends in .cache/replay/<group>.log, a .done marker next to it (.failed when the replay failed: remove it to
# retry). Loops until every group of the window is done (the corpus fill runs meanwhile), so run it detached
# from the deployment root:
#   setsid nohup deploy/replay-window.sh > .cache/replay/driver.log 2>&1 &
# No new group starts while the nightly pause is on (.cache/paused); the running ones finish.
# DB=usenet_rebuild replays into the catalog itself (voyager 3 deployed, 2026-10-26: history comes from the corpus
# this way until the window worker is built); the harness then needs REPLAY_INTO_CATALOG=1, passed here. OUT keeps
# a run's logs and markers apart (.cache/replay-catalog for the catalog's).
set -u
ROOT=${ROOT:-/mnt/bulk/usenet-rebuild}
PRIORITY=${PRIORITY:-0}
PARALLEL=${PARALLEL:-4}
MEMORY=${MEMORY:-12g}
IMAGE=${IMAGE:-usenet-voyager3:latest}
DB=${DB:-usenet_replay}
OUT=${OUT:-$ROOT/.cache/replay}
mkdir -p "$OUT"
cd "$ROOT" || exit 1

say() { echo "$(date -Is) $*"; }
catalog() { docker compose exec -T postgres psql -U rebuild -d usenet_rebuild -X -q -At -F ' ' -c "$1"; }

# The window's groups: name, first article, last article, whether every range is stored; smallest first.
window() {
  catalog "SELECT g.name, min(r.start_id), max(r.end_id), (count(*) = count(r.stored_at))::int
           FROM ranges r JOIN groups g ON g.id = r.group_id
           WHERE r.capture AND r.priority = $PRIORITY
           GROUP BY g.name ORDER BY sum(r.end_id - r.start_id + 1)"
}

replay() {
  local group=$1 from=$2 to=$3
  say "$group: replaying $from–$to"
  if docker run --rm --name "replay-${group//./-}" --network usenet-rebuild --env-file .env \
      -e NODE_ENV=production -e HOST=0.0.0.0 -e PORT=3334 -e APP_URL=http://web:3334 -e SPOOL_DIR=/spool -e CORPUS_DIR=/corpus \
      -e DB_HOST=postgres -e DB_PORT=5432 -e DB_DATABASE="$DB" -e CHUNK_TABLESPACE=pg_default -e REPLAY_INTO_CATALOG=1 \
      -v "$ROOT/corpus:/corpus:ro" --memory "$MEMORY" "$IMAGE" \
      node ace replay:corpus --group "$group" --from "$from" --to "$to" > "$OUT/$group.log" 2>&1; then
    touch "$OUT/$group.done"
    say "$group: done ($(grep -o 'Releases [0-9,]*' "$OUT/$group.log" | tail -1))"
  else
    touch "$OUT/$group.failed"
    say "$group: FAILED (see $OUT/$group.log)"
  fi
  rm -f "$OUT/$group.running"
}

while true; do
  todo=0
  while read -r group from to complete; do
    [ -n "$group" ] || continue
    [ -e "$OUT/$group.done" ] && continue
    [ -e "$OUT/$group.failed" ] && continue
    todo=$((todo + 1))
    [ "$complete" = 1 ] || continue
    [ -e "$OUT/$group.running" ] && continue
    [ -e "$ROOT/.cache/paused" ] && continue
    while [ "$(find "$OUT" -name '*.running' | wc -l)" -ge "$PARALLEL" ]; do sleep 30; done
    touch "$OUT/$group.running"
    replay "$group" "$from" "$to" &
    sleep 5
  done <<< "$(window)"
  if [ "$todo" = 0 ]; then
    wait
    say "every group of the window at priority $PRIORITY is replayed"
    exit 0
  fi
  sleep 300
done