Upload folder using huggingface_hub

OpenMidnight/dinov2/configs/train/openpath_vitg14.yaml

@@ -0,0 +1,95 @@
+# OpenPath — ViT-g/14 pathology FM pre-training config.
+# DINOv2 (DINO + iBOT + KDE) + gram anchoring, warm-started from Meta DINOv2 ViT-g/14-reg,
+# on the public OpenPath corpus (native 40x tiles). Global batch = 64/GPU x 40 GPU = 2560.
+#
+# LR regime: near-constant (flat). `epochs` only sets the schedule horizon (very large), so the
+# cosine barely descends; the run actually stops at `early_stop`. This keeps LR/wd/momentum
+# near their initial values throughout training.
+compute_precision:
+  grad_scaler: false
+  teacher:
+    backbone:
+      sharding_strategy: SHARD_GRAD_OP
+      mixed_precision: {param_dtype: bf16, reduce_dtype: bf16, buffer_dtype: fp32}
+    dino_head:
+      sharding_strategy: SHARD_GRAD_OP
+      mixed_precision: {param_dtype: bf16, reduce_dtype: bf16, buffer_dtype: fp32}
+    ibot_head:
+      sharding_strategy: SHARD_GRAD_OP
+      mixed_precision: {param_dtype: bf16, reduce_dtype: bf16, buffer_dtype: fp32}
+  student:
+    backbone:
+      sharding_strategy: SHARD_GRAD_OP
+      mixed_precision: {param_dtype: bf16, reduce_dtype: bf16, buffer_dtype: fp32}
+    dino_head:
+      sharding_strategy: SHARD_GRAD_OP
+      mixed_precision: {param_dtype: bf16, reduce_dtype: fp32, buffer_dtype: fp32}
+    ibot_head:
+      sharding_strategy: SHARD_GRAD_OP
+      mixed_precision: {param_dtype: bf16, reduce_dtype: fp32, buffer_dtype: fp32}
+dino:
+  head_n_prototypes: 131072
+  head_bottleneck_dim: 384
+  do_kde: True
+  kde_loss_weight: .05
+  koleo_loss_weight: 0
+  do_koleo: False
+ibot:
+  loss_weight: 1.0
+  mask_sample_probability: 0.5
+  mask_ratio_min_max:
+    - 0.1
+    - 0.45
+  separate_head: true
+  head_n_prototypes: 131072
+train:
+  # OpenPath corpus (taejoon89/openpath-corpus). Point to the downloaded shards:
+  sample_list_path: "openpath:glob=/path/to/openpath-corpus/*/tiles/shards/w*/*.tar"
+  streaming_from_hf: false
+  batch_size_per_gpu: 64          # x 40 GPU = global 2560
+  centering: sinkhorn_knopp
+  use_pretrained: True            # Meta DINOv2 ViT-g/14-reg warm-start (set MODEL.WEIGHTS)
+  OFFICIAL_EPOCH_LENGTH: 1250
+  saveckp_freq: 8                 # resume ckpt every 8 x 1250 = 10,000 iters
+  num_workers: 10
+  prefetch_factor: 4
+  skip_checkpointer: false
+student:
+  arch: vit_giant2
+  patch_size: 14
+  drop_path_rate: 0.4
+  ffn_layer: swiglufused
+  block_chunks: 4
+  num_register_tokens: 4
+  interpolate_antialias: true
+  interpolate_offset: 0.0
+teacher:
+  momentum_teacher: 0.994
+  warmup_teacher_temp_epochs: 41  # 0.27 epoch = 51,250 iters
+optim:
+  epochs: 8000                    # schedule horizon only (=10,000,000 iter); LR/wd/momentum near-constant
+  early_stop: 276                 # actual stop: 276 x 1250 = 345,000 iters (~1 native epoch)
+  weight_decay_end: 0.2           # not reached within early_stop (wd stays ~0.040)
+  base_lr: 2.0e-04                # sqrt_wrt_1024 -> effective ~3.16e-4 @ global 2560
+  warmup_epochs: 9                # ~11,250 iter (~4.9%); must be an integer
+  freeze_last_layer_epochs: 1
+  layerwise_decay: 1.0
+crops:
+  local_crops_size: 98
+evaluation:
+  eval_period_iterations: 5750    # save a teacher checkpoint every 5,750 iters
+  bach_root: ""
+  breakhis_root: ""
+  pcam_root: ""
+
+# Gram anchoring (ported from DINOv3): MSE between L2-normalized patch-token Gram matrices of
+# student vs a frozen anchor. Dampens dense-feature degradation during long training.
+gram:
+  use_loss: true
+  normalized: true
+  remove_neg: true
+  loss_weight: 40.0            # ~5-8% of total loss; lower if it over-constrains DINO/iBOT
+  it_first_update: 57500       # activate near the dense-feature peak
+  ramp_iters: 3000             # linear ramp of the gram weight
+  # Anchor = a strong earlier OpenPath teacher checkpoint (see taejoon89/openpath-checkpoints):
+  ckpt: /path/to/openpath-checkpoints/training_63250/teacher_checkpoint.pth

README.md

@@ -1,48 +1,121 @@
-# OpenPath — 병리 Foundation Model 학습 코드 (run6 기준)
+---
+license: apache-2.0
+tags:
+  - pathology
+  - histopathology
+  - foundation-model
+  - self-supervised
+  - dinov2
+  - vision-transformer
+  - digital-pathology
+library_name: pytorch
+pipeline_tag: image-feature-extraction
+---
 
-공개 병리 데이터만으로 학습한 ViT-g/14 pathology FM(OpenPath)의 **학습 재현 코드**.
-run6 = **native 40× 코퍼스 + DINOv2(OpenMidnight fork) + gram anchoring** 구성.
+# OpenPath — Pathology Foundation Model (training code)
+
+**OpenPath** is a vision foundation model for computational pathology: a **ViT-g/14** encoder
+pre-trained with self-supervision (**DINOv2** + **gram anchoring**) on **public-only** whole-slide
+histopathology tiles. This repository contains the **training / reproduction code**. The corpus and
+checkpoints are hosted separately (see below).
+
+- **Encoder:** ViT-g/14 (reg4), 1536-dim CLS embedding
+- **Objective:** DINO + iBOT + KDE (DINOv2) with **gram anchoring** ported from DINOv3
+- **Data:** public pathology WSIs only (TCGA, TCIA, GTEx, CAMELYON, ACROBAT, SurGen, …), re-tiled at native 40×
+- **Warm start:** Meta DINOv2 ViT-g/14-reg
+- **Training:** FSDP (SHARD_GRAD_OP), bf16, flat learning-rate schedule, 40× B200 (multi-node)
+
+## Repository layout
 
-## 구성
 ```
-OpenMidnight/          # DINOv2 학습 fork (dinov2 패키지 + 우리 수정)
-  dinov2/train/train.py            # 학습 루프 (+ gram weight 스케줄)
-  dinov2/train/ssl_meta_arch.py    # SSL 아키텍처 (+ gram anchor teacher)
-  dinov2/loss/gram_loss.py         # ★ gram anchoring loss (DINOv3서 이식)
-  dinov2/data/openpath_wds.py      # openpath 코퍼스 WebDataset 로더 (openpath:glob=...)
-  dinov2/configs/train/openpath_vitg14_run6_native_gram.yaml   # ★ run6 config
+OpenMidnight/                                  # DINOv2 training fork
+  dinov2/train/train.py                        # training loop (+ gram-weight schedule)
+  dinov2/train/ssl_meta_arch.py                # SSL arch (+ frozen gram-anchor teacher)
+  dinov2/loss/gram_loss.py                     # gram anchoring loss (ported from DINOv3)
+  dinov2/data/openpath_wds.py                  # WebDataset loader for the OpenPath corpus
+  dinov2/configs/train/openpath_vitg14.yaml    # training config
 scripts/
-  launch_om_run2.sh      # 멀티노드 런처 (40 GPU: host + 4 workers)
-  autoresume_run6.sh     # 크래시 내성 자동 재개
-  watch_eval_run6.sh     # 실시간 HEST probe (체크포인트별)
-  run_hest_3way.py       # HEST 평가 (Meta DINOv2 / Phikon-v2 / OpenPath 비교)
+  launch.sh                  # multi-node launcher (host + workers, 40 GPU)
+  autoresume.sh              # crash-tolerant auto-resume
+  watch_eval.sh              # online HEST probing per checkpoint
+  run_hest_3way.py           # HEST evaluation (Meta DINOv2 / Phikon-v2 / OpenPath)
 requirements.txt
 ```
 
-## 데이터
-- **`taejoon89/openpath-corpus`** (HF, public+gated) — native 40× 병리 타일 33,991 shard / 17TB.
-- config의 `train.sample_list_path` = `openpath:glob=<코퍼스경로>/*/tiles/shards/w*/*.tar` 로 지정.
+## Related artifacts
+
+| Artifact | Hugging Face repo | Notes |
+|---|---|---|
+| **Corpus** | `taejoon89/openpath-corpus` | Native 40× pathology tiles, 33,991 WebDataset shards / ~17 TB (public, gated) |
+| **Checkpoints** | `taejoon89/openpath-checkpoints` | OpenPath teacher checkpoints across training (private) |
+| **Code** | `taejoon89/openpath` | This repository |
+
+The training config points to the corpus via
+`train.sample_list_path: "openpath:glob=<corpus>/*/tiles/shards/w*/*.tar"`. The gram anchor
+(`gram.ckpt`) is an earlier OpenPath teacher checkpoint from `openpath-checkpoints`.
+
+## Method — gram anchoring
+
+Long self-supervised training degrades dense/patch features. Following **DINOv3**, we add a
+**gram anchoring** loss: the MSE between the L2-normalized patch-token Gram (similarity) matrices
+of the student and a **frozen anchor** model (a strong earlier checkpoint). The loss weight is `40`
+and it activates near the dense-feature peak (iteration `57,500`) with a 3k-iter ramp. This
+**dampens the post-peak decline** of dense representations while DINO/iBOT keep optimizing the
+global representation.
+
+## Key hyper-parameters
 
-## 앵커 (gram)
-- run6의 gram anchor = **run5 training_63250** teacher checkpoint (HEST 0.3834).
-- `taejoon89/openpath-checkpoints` (HF, private)에서 받아 config의 `gram.ckpt`에 지정.
+| | |
+|---|---|
+| Arch | `vit_giant2`, patch 14, 4 register tokens, SwiGLU FFN |
+| Batch | 64 / GPU × 40 GPU = global 2560 |
+| LR | base 2e-4 (effective ≈ 3.16e-4 @ global 2560), flat (near-constant) |
+| Schedule | `epochs: 8000` horizon, `early_stop: 276` ≈ 345k iters ≈ 1 native epoch |
+| gram | weight 40, `it_first_update 57500`, ramp 3000, normalized, remove-neg |
+| Precision | bf16, FSDP SHARD_GRAD_OP, sinkhorn-knopp centering |
 
-## 핵심 설정 (run6)
-- ViT-g/14 reg4, patch14, warm-start = Meta DINOv2 ViT-g/14-reg.
-- flat-LR (epochs 8000 지평, early_stop 276=345k=native 1 epoch), base_lr 2e-4 (실효 3.16e-4 @global 2560).
-- **gram anchoring**: weight 40, it_first_update 57500(peak 직전 활성), ramp 3000. loss = 정규화 patch Gram matrix MSE(student vs anchor).
-- FSDP SHARD_GRAD_OP, bf16, sinkhorn-knopp centering, DINO+iBOT+KDE loss.
+## Reproducing training
 
-## 학습 실행 (멀티노드 40 GPU 예)
 ```bash
-export PYTHONPATH=$PWD/OpenMidnight
-CFG=OpenMidnight/dinov2/configs/train/openpath_vitg14_run6_native_gram.yaml
-bash scripts/launch_om_run2.sh omvitgrun6 "$CFG" <출력디렉토리> <로그디렉토리>
-# + autoresume_run6.sh (백그라운드), watch_eval_run6.sh (실시간 HEST)
+export PYTHONPATH="$PWD/OpenMidnight"
+CFG=OpenMidnight/dinov2/configs/train/openpath_vitg14.yaml
+# edit CFG: train.sample_list_path (corpus glob), gram.ckpt (anchor checkpoint), MODEL.WEIGHTS (DINOv2 warm-start)
+bash scripts/launch.sh openpathrun "$CFG" <output_dir> <log_dir>
+# optionally run scripts/autoresume.sh (background) and scripts/watch_eval.sh (online HEST)
 ```
 
-## 결과 (오염-free HCC-ST Visium 벤치, LOPO Pearson)
-- best 모델 = run5 training_63250: HEST 0.3834, HCC-ST 0.3244 (> OpenMidnight 0.3004).
-- run6(gram) = 후반 하락 완화(안정화) + dense feature. HCC-ST 대부분 체크포인트가 OpenMidnight 상회.
+Extract CLS embeddings for downstream use:
+
+```python
+import torch, dinov2.models.vision_transformer as vits
+ck = torch.load("teacher_checkpoint.pth", map_location="cpu", weights_only=False)
+sd = {k[len("backbone."):]: v for k, v in ck["teacher"].items() if k.startswith("backbone.")}
+m = vits.vit_giant2(patch_size=14, img_size=224, block_chunks=4, num_register_tokens=4,
+                    ffn_layer="swiglufused", init_values=1e-5,
+                    interpolate_antialias=True, interpolate_offset=0.0)
+m.load_state_dict(sd, strict=True); m.eval()
+cls = m.forward_features(x)["x_norm_clstoken"]   # (B, 1536)
+```
+
+## Evaluation
+
+Frozen-encoder linear/ridge probing. The headline benchmark is a **contamination-free** in-house
+HCC (hepatocellular carcinoma) Visium spatial-transcriptomics cohort (leave-one-patient-out, mean
+Pearson over top-50 highly-variable genes) — no public FM was trained on it.
+
+| Benchmark (metric) | OpenPath (best) | OpenMidnight | Phikon-v2 |
+|---|---|---|---|
+| **HCC-ST** (Pearson, clean) | **0.324** | 0.300 | 0.274 |
+| HEST-1K (Pearson) | 0.383 | 0.390 | 0.375 |
+| NCT-CRC-HE (9-class acc) | 0.964 | 0.967 | 0.937 |
+| BACH (4-class acc) | 0.811 | 0.906 | 0.708 |
+
+On the clean HCC-ST cohort the majority of OpenPath checkpoints exceed OpenMidnight, and OpenPath
+beats Phikon-v2 across all benchmarks. The best checkpoint (`training_63250`, available in
+`openpath-checkpoints`) is top on both HEST-1K and HCC-ST. Note: PCam / CAMELYON is excluded from
+comparison because it overlaps our training corpus.
+
+## License
 
-라이선스: 코드 = OpenMidnight/DINOv2 fork(해당 LICENSE 참조). 데이터 = 공개 병리(CC-BY/CC0/NIH-open).
+Code: derived from the OpenMidnight / DINOv2 fork — see `OpenMidnight/LICENSE`.
+Training data: public pathology datasets under CC-BY / CC0 / NIH-open terms (redistributable).

scripts/autoresume.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# run(om_vitg14_run) 크래시 내성 자동 재개 waiter. TJ-1(오케스트레이터)서 백그라운드 실행.
+#  - 완주(training_230000) 감지 → exit 0
+#  - NaN 감지 → exit 2 (재개 안 함)
+#  - 크래시(전멸, 완주/NaN 아님) → 전노드 shm정리 후 launch.sh로 resume(같은 OUT=마지막 체크포인트), 최대 MAX회
+# ★ host=TJ-2(alive/NaN 체크도 TJ-2). TJ-1은 학습 제외라 대상 아님.
+set -u
+P=/NHNHOME/WORKSPACE/0526040027_A/OpenPath
+CFG=$P/OpenMidnight/dinov2/configs/train/openpath_vitg14.yaml
+OUT=$P/data/runs/om_vitg14_run
+LOG=$P/data/runs/om_logs/vitg_run
+DONE=$OUT/eval/training_345000/teacher_checkpoint.pth
+TJLOG=$LOG/TJ-2.log
+NODES="TJ-2 TJ-3 TJ-4 TJ-5 TJ-6"
+MAX=40
+retry=0
+
+alive_count() {
+  timeout 12 ssh -o BatchMode=yes -o ConnectTimeout=8 TJ-2 "ps -eo args 2>/dev/null | grep '[t]rain.py' | wc -l" 2>/dev/null || echo 1
+}
+
+echo "[autoresume-run] watching $OUT (host TJ-2, max $MAX auto-resumes)"
+while true; do
+  [ -f "$DONE" ] && { echo "RUN2_DONE: 완주 (training_230000)"; exit 0; }
+  if grep -qiE "NaN detected" "$TJLOG" 2>/dev/null; then echo "RUN2_NAN: NaN 감지 — 자동재개 중단"; exit 2; fi
+
+  a=$(alive_count)
+  if [ "${a:-1}" = "0" ]; then
+    sleep 60
+    a2=$(alive_count)
+    if [ "${a2:-1}" = "0" ] && [ ! -f "$DONE" ]; then
+      grep -qiE "NaN detected" "$TJLOG" 2>/dev/null && { echo "RUN2_NAN"; exit 2; }
+      retry=$((retry+1))
+      if [ "$retry" -gt "$MAX" ]; then echo "RUN2_GIVEUP: 자동재개 $MAX회 초과 — 수동 개입"; exit 3; fi
+      lastit=$(grep -E "helpers.py:110] Training" "$TJLOG" 2>/dev/null | tail -1 | grep -oE "\[ *[0-9]+/" | head -1)
+      echo "[autoresume-run] CRASH detected at $lastit — auto-resume #$retry/$MAX"
+      for n in $NODES; do timeout 10 ssh -o BatchMode=yes "$n" "pkill -9 -f 'dinov2/train/train.py' 2>/dev/null; pkill -9 -f torchrun 2>/dev/null; rm -f /dev/shm/nccl* 2>/dev/null" 2>/dev/null; done
+      sleep 30
+      bash "$P/scripts/launch.sh" "openpathrunr${retry}" "$CFG" "$OUT" "$LOG" >/dev/null 2>&1
+      echo "[autoresume-run] relaunched (rdzv openpathrunr${retry}) — resume from last checkpoint"
+      sleep 180
+    fi
+  fi
+  sleep 240
+done

scripts/launch.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# run 학습 런처: TJ-2~6(5노드×8=40 GPU) FSDP. host=TJ-2, 워커=TJ-3~6. ★TJ-1은 학습 제외(eval watcher 전용).
+# TJ-1(오케스트레이터)에서 실행 — 로컬 학습 프로세스 없음. host도 원격(ssh TJ-2)이라 자기참조 함정 없음.
+# 사용: scripts/launch.sh <rdzv_id> <config_abspath> <outdir> <logdir> [extra train.py args...]
+set -u
+R=/NHNHOME/WORKSPACE/0526040027_A/OpenPath
+RDZV="$1"; CFG="$2"; OUTDIR="$3"; LOGDIR="$4"; shift 4
+EXTRA="$*"
+HOST="TJ-2"
+EP="10.34.15.21:29500"           # TJ-2 IB IP (rendezvous rank0)
+WORKERS="TJ-3 TJ-4 TJ-5 TJ-6"
+PYC="\$HOME/.cache/op_pyc_$RDZV"  # 노드-로컬. 원격 셸에서 확장되도록 escape
+ENV="WANDB_MODE=disabled NCCL_DEBUG=WARN NCCL_IB_HCA=mlx5_1,mlx5_2,mlx5_4,mlx5_9 \
+NCCL_SOCKET_IFNAME=bond0 GLOO_SOCKET_IFNAME=bond0 \
+NCCL_IB_TIMEOUT=22 NCCL_IB_RETRY_CNT=13 NCCL_IB_QPS_PER_CONNECTION=4 \
+PYTHONDONTWRITEBYTECODE=1 PYTHONPYCACHEPREFIX=$PYC PYTHONPATH=$R/OpenMidnight"
+TR="venv/bin/torchrun --nnodes=5 --nproc-per-node=8 --rdzv-backend=c10d \
+--rdzv-endpoint=$EP --rdzv-id=$RDZV \
+OpenMidnight/dinov2/train/train.py --config-file $CFG --output-dir $OUTDIR $EXTRA"
+
+mkdir -p "$LOGDIR" "$OUTDIR"
+
+# 1) host(TJ-2) 원격 기동 — ★ssh를 &로 백그라운드(안 그러면 원격 setsid의 상속 fd를 물고 ssh가 반환 안 함=런처 행)
+echo "launching host on $HOST ..."
+ssh -n -o BatchMode=yes "$HOST" "mkdir -p $PYC; cd $R && setsid env $ENV $TR > $LOGDIR/${HOST}.log 2>&1 < /dev/null & echo ${HOST}-host-fired" &
+
+# 2) host torchrun가 store 바인딩할 시간(고정 대기). c10d는 워커가 재시도로 붙으므로 ss폴링 불필요.
+echo "waiting 30s for host store bind on $EP ..."
+sleep 30
+echo "firing 4 workers"
+
+# 3) 워커(TJ-3~6) 병렬 기동
+for n in $WORKERS; do
+  ssh -n -o BatchMode=yes "$n" "mkdir -p $PYC; cd $R && setsid env $ENV $TR > $LOGDIR/${n}.log 2>&1 < /dev/null & echo ${n}-fired" &
+done
+# ★ wait 대신 고정 sleep — bg ssh가 원격 setsid의 fd를 물어 반환 안 할 수 있어 wait는 영원히 멈춤(autoresume blocking 호출도 막힘).
+sleep 20
+echo "ALL launched: host $HOST + workers $WORKERS | rdzv=$RDZV | logs=$LOGDIR | out=$OUTDIR"

scripts/run_hest_3way.py

@@ -4,7 +4,7 @@
 benchmark()는 resnet50을 자동 기준으로 포함하므로 출력에 resnet50도 함께 나온다.
 
 실행(venv_hest):
-  PYTHONPATH=third_party/OpenMidnight:. venv_hest/bin/python scripts/run_hest_3way.py \
+  PYTHONPATH=OpenMidnight:. venv_hest/bin/python scripts/run_hest_3way.py \
      --backbone om --weights data/runs/om_vitl14/eval/training_0/teacher_checkpoint.pth \
      --exp-code meta_dinov2_vitl14   [task1 ...]
   PYTHONPATH=. venv_hest/bin/python scripts/run_hest_3way.py --backbone phikon --exp-code phikon_v2
@@ -29,7 +29,7 @@ eval_tf = transforms.Compose([
 
 def build_om(weights):
     """OpenMidnight teacher_checkpoint(ViT-L/14 reg4) → CLS nn.Module."""
-    _OM = f"{ROOT}/third_party/OpenMidnight"
+    _OM = f"{ROOT}/OpenMidnight"
     sys.path = [_OM] + [p for p in sys.path if "third_party/dinov2" not in p]
     import dinov2.models.vision_transformer as vits
     ck = torch.load(weights, map_location="cpu", weights_only=False)

scripts/watch_eval.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# TJ-1 실시간 HEST eval watcher. run 학습이 뱉는 teacher_checkpoint를 감지→probe→곡선 append.
+# TJ-1 로컬 실행(venv_hest·GPU 로컬). 1개씩 순차(동시 HDF5 버그 회피, ~60min<체크포인트간격 ~84min라 페이스 유지).
+# per-exp embed 분리(/dev/shm/hest_emb_<N>), pkill 미포함(self-kill 회피).
+set -u
+R=/NHNHOME/WORKSPACE/0526040027_A/OpenPath
+OUTDIR=$R/data/runs/om_vitg14_run
+EVALDIR=$OUTDIR/eval
+RESDIR=$R/data/runs/hest_3way
+CURVE=$RESDIR/curve.txt
+BENCH=/dev/shm/hest_bench
+GPU=0
+mkdir -p "$RESDIR"
+
+# hest_bench를 TJ-1 /dev/shm에 확보(없으면 Lustre서 복사, 40G)
+if [ ! -d "$BENCH" ] || [ "$(ls "$BENCH" 2>/dev/null | wc -l)" -lt 5 ]; then
+  echo "[watch] copying hest_bench -> $BENCH (40G, 수분) ..."
+  rm -rf "$BENCH"; cp -r "$R/data/eva/hest_bench" "$BENCH"
+fi
+echo "[watch] START — watching $EVALDIR (TJ-1 GPU$GPU, 1-at-a-time) | curve=$CURVE"
+
+cd "$R"
+while true; do
+  for ck in $(ls -d "$EVALDIR"/training_* 2>/dev/null | sort -t_ -k2 -n); do
+    N=$(basename "$ck" | sed 's/training_//')
+    W="$ck/teacher_checkpoint.pth"
+    LOG="$RESDIR/run_$N.log"
+    [ -f "$W" ] || continue
+    grep -q "per-encoder avg Pearson" "$LOG" 2>/dev/null && continue   # 이미 완료
+    EMB="/dev/shm/hest_emb_$N"
+    rm -rf "$EMB"
+    echo "[watch] $(date '+%F %T') probing training_$N ..."
+    HEST_EMBED_ROOT="$EMB" HEST_BENCH_ROOT="$BENCH" HDF5_USE_FILE_LOCKING=FALSE \
+      CUDA_VISIBLE_DEVICES=$GPU PYTHONPATH=OpenMidnight:. \
+      venv_hest/bin/python -u scripts/run_hest_3way.py --backbone om --weights "$W" --exp-code "run_$N" > "$LOG" 2>&1
+    rm -rf "$EMB"
+    ce=$(grep "per-encoder avg Pearson" "$LOG" 2>/dev/null | tail -1 | grep -oE "'custom_encoder': [0-9.]+" | grep -oE "[0-9.]+")
+    rn=$(grep "per-encoder avg Pearson" "$LOG" 2>/dev/null | tail -1 | grep -oE "'resnet50': [0-9.]+" | grep -oE "[0-9.]+")
+    printf "%s\tours=%s\tresnet50=%s\t%s\n" "training_$N" "${ce:-FAIL}" "${rn:-NA}" "$(date '+%F %T')" >> "$CURVE"
+    echo "[watch] training_$N = ${ce:-FAIL}"
+  done
+  sleep 120
+done