harness/scripts/status.sh

#!/bin/bash
# One-screen status: what is running, what exists, where the numbers are.
WS=/mnt/pvc/users/simon/agentptb/runs/a-opus-max/workspace
cd "$WS" || exit 1
echo "=== time ==="
date -u '+%Y-%m-%dT%H:%M:%SZ  now'
if [ -f DEADLINE ]; then
  END=$(cut -d. -f1 < DEADLINE)
  printf 'deadline %s   remaining %s h\n' "$(date -u -d "@$END" '+%Y-%m-%dT%H:%M:%SZ')" \
      "$(python3 -c "print(round(($END - $(date +%s))/3600, 1))")"
fi

echo; echo "=== processes ==="
pgrep -af "PRIME-RL|vllm serve|bin/eval|bin/rl|pool_probe.sh" 2>/dev/null \
  | grep -v "bash -c" | sed 's/\(.\{110\}\).*/\1…/' | head -12
[ -z "$(pgrep -f 'PRIME-RL|vllm serve|bin/eval')" ] && echo "(no training/eval/serving running)"

echo; echo "=== gpus (used vs my processes — a gap means a neighbour holds the card) ==="
nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader | head -6
nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader | head -6

echo; echo "=== training ==="
for f in logs/sft_*.log logs/rl_*.log; do
  [ -f "$f" ] || continue
  last=$(grep -oE "Step [0-9]+ \|[^|]*\| Loss [0-9.]+" "$f" | tail -1)
  [ -n "$last" ] && echo "$f: $last"
done
echo "checkpoints:"; ls -d ckpt/*/weights/step_* 2>/dev/null | tail -6

echo; echo "=== sandbox pool ==="
tail -2 logs/pool_probe.log 2>/dev/null

echo; echo "=== scored runs ==="
for d in runs/*/; do
  [ -f "$d/traces.jsonl" ] || continue
  python3 scripts/score.py "$d" 2>/dev/null | sed -n '1,2p' | paste -sd' ' -
done