src/status.sh

#!/usr/bin/env bash
# User is in IST (UTC+5:30) -- report wall-clock in IST, not UTC.
export TZ="Asia/Kolkata"
# One-line-ish status snapshot for periodic reporting.
cd /workspace/creative-writing
STAGE=$(grep -aE "^== (START|OK|FAIL|ABORT)" logs/pipeline.log 2>/dev/null | tail -1 | sed 's/  */ /g')

# active training log (most recently modified)
# only consider logs of a CURRENTLY RUNNING job; otherwise a finished
# stage's log masquerades as live status (this bit me once already).
RUNCFG=$(pgrep -af "src/train_(grpo|dpo).py" | grep -oE "configs/[^ ]+\.yaml" | head -1)
ACT=""
if [ -n "$RUNCFG" ]; then
  NM=$(python3 -c "import yaml,sys;print(yaml.safe_load(open(sys.argv[1]))['name'])" "$RUNCFG" 2>/dev/null)
  for c in "logs/train_${NM}.log" "logs/lrprobe_3e-5.log" $(ls -t logs/*.log 2>/dev/null); do
    [ -f "$c" ] && ACT="$c" && break
  done
fi
PROG=""; RW=""; ENT=""
if [ -n "$ACT" ]; then
  PROG=$(tr '\r' '\n' < "$ACT" | grep -aE "^ *[0-9]+%\|" | tail -1 | sed 's/  */ /g')
  RW=$(grep -ao "\[rw\].*" "$ACT" 2>/dev/null | tail -1)
  ENT=$(grep -aoE "'entropy': '[^']*'" "$ACT" 2>/dev/null | tail -1)
fi

VRAM=$(nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader,nounits | tr ',' '/' | tr -d ' ')
DISK=$(df -BG --output=avail /workspace | tail -1 | tr -d ' ')
SPEND=$(python3 -c "
import os,urllib.request,json
try:
    r=urllib.request.Request('https://openrouter.ai/api/v1/key',headers={'Authorization':'Bearer '+os.environ['OPENROUTER_API_KEY']})
    d=json.load(urllib.request.urlopen(r,timeout=20))['data']
    print(f\"\${d['usage']:.2f}used/\${d['limit_remaining']:.2f}left\")
except Exception: print('n/a')" 2>/dev/null)

ERR=$(grep -alE "OutOfMemory|Traceback|UNHEALTHY|GUARDRAIL" logs/train_*.log logs/E[34]*.log logs/eval_all.log 2>/dev/null | tr '\n' ',' )

NOW=$(TZ=Asia/Kolkata date +"%H:%M IST")
echo "[$NOW] STAGE=[$STAGE] PROG=[$PROG] ${RW:+RW=[$RW] }${ENT:+$ENT }vram=${VRAM} disk=${DISK} judge=${SPEND}${ERR:+ ERRORS_IN=$ERR}"