tools/watch_ingest.sh

#!/usr/bin/env bash
# Re-ingest a live sweep into the viewer every INTERVAL seconds, so progress is visible while it runs
# rather than only at the end. The eval tool writes partial summaries atomically, so reading one mid-run
# is safe; a half-written file would have looked like a corrupt result instead of an in-progress one.
set -uo pipefail
VIZ="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PY="${PY:-/fsx/adithyaskolavi/projects/trl_prod/.venv312/bin/python}"
SRC="${SRC:?set SRC=/path/to/sweep.json}"
DSID="${DSID:-phase0}"
INTERVAL="${INTERVAL:-120}"
JOB="${JOB:-}"
LOG="$VIZ/logs/watch_ingest.log"; mkdir -p "$VIZ/logs"

log(){ echo "$(date -Is) $*" >> "$LOG"; }
log "watching $SRC -> dataset $DSID every ${INTERVAL}s (job ${JOB:-none})"

while true; do
  if [ -f "$SRC" ]; then
    if "$PY" "$VIZ/tools/ingest.py" eval --json "$SRC" --dataset-id "$DSID" \
         --dataset-label "phase 0 · 15 harnesses · 50 tasks · k=4" \
         --project data-agent --tasks-from AdithyaSK/data_agent_rl_environment_eval \
         --notes "live sweep" >>"$LOG" 2>&1; then
      # The viewer caches listings, so tell it to drop them; otherwise a refresh in the browser shows
      # the same numbers for up to REFRESH_TTL seconds and looks stuck.
      curl -fs -m 10 -X POST http://127.0.0.1:8090/api/refresh >/dev/null 2>&1 || true
      log "ingested $(stat -c%s "$SRC") bytes"
    else
      log "ingest failed (likely a partial write mid-move); retrying next tick"
    fi
  else
    log "waiting for $SRC to appear"
  fi
  # Stop once the job is gone AND one final ingest has run, so the last state is always captured.
  if [ -n "$JOB" ] && ! squeue -h -j "$JOB" >/dev/null 2>&1; then
    log "job $JOB finished; final ingest done, exiting"
    exit 0
  fi
  sleep "$INTERVAL"
done