scripts/watch_w1.sh

#!/usr/bin/env bash
# Close supervision for the W=1 transfer run.
# Health signals:
#   OK  — stages advancing, hops 1..6 stay >= 0.95, rare/no backtrack to early stages
#   BAD — frequent target_stage in {0..5}, hops 1..6 collapsing, stuck repairing early
set -uo pipefail
cd /egr/research-slim/ghoshavr/reasoning-by-superposition-main
LOG=logs/L15_push_2L_ce95_100k_w1.log
OUT=logs/watch_w1_status.txt
INTERVAL=${1:-120}

while true; do
  {
    echo "===== $(date '+%Y-%m-%d %H:%M:%S') ====="
    if ! pgrep -f '[r]un.py args/L15_push_2L_ce95_100k_w1' >/dev/null; then
      echo "PROCESS DEAD"
      grep -E 'Traceback|Error|SignalException' "$LOG" | tail -5
      echo
      sleep "$INTERVAL"
      continue
    fi
    ep=$(grep -oE 'train epoch [0-9]+' "$LOG" 2>/dev/null | tail -1 | awk '{print $3}')
    echo "epoch=${ep:-?}  gpu2=$(nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader -i 2 2>/dev/null)"
    echo "--- latest stage decision ---"
    grep '\[acc-stage\]' "$LOG" | tail -2
    echo "--- latest backtrack ---"
    grep 'target_stage=' "$LOG" | tail -3
    echo "--- latest ce_score (hops 1-7 matter most) ---"
    grep -E 'frontier=\[' "$LOG" | tail -1 | sed 's/.*ce_score=/ce_score=/'
    # count backtracks into early hops (0..5) vs none
    early_bt=$(grep -cE 'target_stage=[0-5]\b' "$LOG" 2>/dev/null || echo 0)
    none_bt=$(grep -c 'target_stage=None' "$LOG" 2>/dev/null || echo 0)
    prom=$(grep -c 'PROMOTE' "$LOG" 2>/dev/null || echo 0)
    echo "promotes=$prom  backtrack_none=$none_bt  backtrack_early(0-5)=$early_bt"
    if grep -qE 'Traceback|CUDA out of memory' "$LOG"; then
      echo "ALERT: error in log"
      grep -E 'Traceback|CUDA out of memory' "$LOG" | tail -3
    fi
    echo
  } | tee -a "$OUT"
  sleep "$INTERVAL"
done