File size: 2,113 Bytes
97863b7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | #!/usr/bin/env bash
# TJ-1 실시간 HEST eval watcher. run6 학습이 뱉는 teacher_checkpoint를 감지→probe→곡선 append.
# TJ-1 로컬 실행(venv_hest·GPU 로컬). 1개씩 순차(동시 HDF5 버그 회피, ~60min<체크포인트간격 ~84min라 페이스 유지).
# per-exp embed 분리(/dev/shm/hest_emb_run6_<N>), pkill 미포함(self-kill 회피).
set -u
R=/NHNHOME/WORKSPACE/0526040027_A/OpenPath
OUTDIR=$R/data/runs/om_vitg14_run6
EVALDIR=$OUTDIR/eval
RESDIR=$R/data/runs/hest_3way
CURVE=$RESDIR/run6_curve.txt
BENCH=/dev/shm/hest_bench
GPU=0
mkdir -p "$RESDIR"
# hest_bench를 TJ-1 /dev/shm에 확보(없으면 Lustre서 복사, 40G)
if [ ! -d "$BENCH" ] || [ "$(ls "$BENCH" 2>/dev/null | wc -l)" -lt 5 ]; then
echo "[watch] copying hest_bench -> $BENCH (40G, 수분) ..."
rm -rf "$BENCH"; cp -r "$R/data/eva/hest_bench" "$BENCH"
fi
echo "[watch] START — watching $EVALDIR (TJ-1 GPU$GPU, 1-at-a-time) | curve=$CURVE"
cd "$R"
while true; do
for ck in $(ls -d "$EVALDIR"/training_* 2>/dev/null | sort -t_ -k2 -n); do
N=$(basename "$ck" | sed 's/training_//')
W="$ck/teacher_checkpoint.pth"
LOG="$RESDIR/run6_$N.log"
[ -f "$W" ] || continue
grep -q "per-encoder avg Pearson" "$LOG" 2>/dev/null && continue # 이미 완료
EMB="/dev/shm/hest_emb_run6_$N"
rm -rf "$EMB"
echo "[watch] $(date '+%F %T') probing training_$N ..."
HEST_EMBED_ROOT="$EMB" HEST_BENCH_ROOT="$BENCH" HDF5_USE_FILE_LOCKING=FALSE \
CUDA_VISIBLE_DEVICES=$GPU PYTHONPATH=third_party/OpenMidnight:. \
venv_hest/bin/python -u scripts/run_hest_3way.py --backbone om --weights "$W" --exp-code "run6_$N" > "$LOG" 2>&1
rm -rf "$EMB"
ce=$(grep "per-encoder avg Pearson" "$LOG" 2>/dev/null | tail -1 | grep -oE "'custom_encoder': [0-9.]+" | grep -oE "[0-9.]+")
rn=$(grep "per-encoder avg Pearson" "$LOG" 2>/dev/null | tail -1 | grep -oE "'resnet50': [0-9.]+" | grep -oE "[0-9.]+")
printf "%s\tours=%s\tresnet50=%s\t%s\n" "training_$N" "${ce:-FAIL}" "${rn:-NA}" "$(date '+%F %T')" >> "$CURVE"
echo "[watch] training_$N = ${ce:-FAIL}"
done
sleep 120
done
|