Upload folder using huggingface_hub

README.md

@@ -95,6 +95,8 @@ global representation.
 export PYTHONPATH="$PWD/OpenPath"
 CFG=OpenPath/dinov2/configs/train/openpath_vitg14.yaml
 # edit CFG: train.sample_list_path (corpus glob), gram.ckpt (anchor checkpoint), MODEL.WEIGHTS (DINOv2 warm-start)
+# set your cluster (see scripts/launch.sh header): MASTER_NODE, WORKER_NODES, MASTER_ADDR, NCCL_IB_HCA, *_SOCKET_IFNAME
+export MASTER_NODE=node1 WORKER_NODES="node2 node3 node4 node5" MASTER_ADDR=<master-ib-ip>
 bash scripts/launch.sh openpathrun "$CFG" <output_dir> <log_dir>
 # optionally run scripts/autoresume.sh (background) and scripts/watch_eval.sh (online HEST)
 ```

scripts/autoresume.sh

@@ -3,38 +3,39 @@
 #  - 완주(training_345000) 감지 → exit 0
 #  - NaN 감지 → exit 2 (재개 안 함)
 #  - 크래시(전멸, 완주/NaN 아님) → 전노드 shm정리 후 launch.sh로 resume(같은 OUT=마지막 체크포인트), 최대 MAX회
-# ★ host=TJ-2(alive/NaN 체크도 TJ-2). TJ-1은 학습 제외라 대상 아님.
+# ★ 인프라는 env로 설정: MASTER_NODE(alive/NaN 체크 대상 = launch.sh의 host), TRAIN_NODES(정리 대상 전 노드).
 set -u
-P=/NHNHOME/WORKSPACE/0526040027_A/OpenPath
+P="${OPENPATH_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}"
+MASTER_NODE="${MASTER_NODE:-node1}"                 # launch.sh의 host 노드
+TRAIN_NODES="${TRAIN_NODES:-node1 node2 node3 node4 node5}"
 CFG=$P/OpenPath/dinov2/configs/train/openpath_vitg14.yaml
 OUT=$P/data/runs/openpath_run
 LOG=$P/data/runs/logs/openpath_run
 DONE=$OUT/eval/training_345000/teacher_checkpoint.pth
-TJLOG=$LOG/TJ-2.log
-NODES="TJ-2 TJ-3 TJ-4 TJ-5 TJ-6"
+HOSTLOG=$LOG/${MASTER_NODE}.log
 MAX=40
 retry=0
 
 alive_count() {
-  timeout 12 ssh -o BatchMode=yes -o ConnectTimeout=8 TJ-2 "ps -eo args 2>/dev/null | grep '[t]rain.py' | wc -l" 2>/dev/null || echo 1
+  timeout 12 ssh -o BatchMode=yes -o ConnectTimeout=8 "$MASTER_NODE" "ps -eo args 2>/dev/null | grep '[t]rain.py' | wc -l" 2>/dev/null || echo 1
 }
 
-echo "[autoresume-run] watching $OUT (host TJ-2, max $MAX auto-resumes)"
+echo "[autoresume-run] watching $OUT (host $MASTER_NODE, max $MAX auto-resumes)"
 while true; do
   [ -f "$DONE" ] && { echo "DONE: 완주 (training_345000)"; exit 0; }
-  if grep -qiE "NaN detected" "$TJLOG" 2>/dev/null; then echo "NAN: NaN 감지 — 자동재개 중단"; exit 2; fi
+  if grep -qiE "NaN detected" "$HOSTLOG" 2>/dev/null; then echo "NAN: NaN 감지 — 자동재개 중단"; exit 2; fi
 
   a=$(alive_count)
   if [ "${a:-1}" = "0" ]; then
     sleep 60
     a2=$(alive_count)
     if [ "${a2:-1}" = "0" ] && [ ! -f "$DONE" ]; then
-      grep -qiE "NaN detected" "$TJLOG" 2>/dev/null && { echo "NAN"; exit 2; }
+      grep -qiE "NaN detected" "$HOSTLOG" 2>/dev/null && { echo "NAN"; exit 2; }
       retry=$((retry+1))
       if [ "$retry" -gt "$MAX" ]; then echo "GIVEUP: 자동재개 $MAX회 초과 — 수동 개입"; exit 3; fi
-      lastit=$(grep -E "helpers.py:110] Training" "$TJLOG" 2>/dev/null | tail -1 | grep -oE "\[ *[0-9]+/" | head -1)
+      lastit=$(grep -E "helpers.py:110] Training" "$HOSTLOG" 2>/dev/null | tail -1 | grep -oE "\[ *[0-9]+/" | head -1)
       echo "[autoresume-run] CRASH detected at $lastit — auto-resume #$retry/$MAX"
-      for n in $NODES; do timeout 10 ssh -o BatchMode=yes "$n" "pkill -9 -f 'dinov2/train/train.py' 2>/dev/null; pkill -9 -f torchrun 2>/dev/null; rm -f /dev/shm/nccl* 2>/dev/null" 2>/dev/null; done
+      for n in $TRAIN_NODES; do timeout 10 ssh -o BatchMode=yes "$n" "pkill -9 -f 'dinov2/train/train.py' 2>/dev/null; pkill -9 -f torchrun 2>/dev/null; rm -f /dev/shm/nccl* 2>/dev/null" 2>/dev/null; done
       sleep 30
       bash "$P/scripts/launch.sh" "openpathrunr${retry}" "$CFG" "$OUT" "$LOG" >/dev/null 2>&1
       echo "[autoresume-run] relaunched (rdzv openpathrunr${retry}) — resume from last checkpoint"

scripts/launch.sh

@@ -1,38 +1,42 @@
 #!/usr/bin/env bash
-# run 학습 런처: TJ-2~6(5노드×8=40 GPU) FSDP. host=TJ-2, 워커=TJ-3~6. ★TJ-1은 학습 제외(eval watcher 전용).
-# TJ-1(오케스트레이터)에서 실행 — 로컬 학습 프로세스 없음. host도 원격(ssh TJ-2)이라 자기참조 함정 없음.
+# Multi-node training launcher (FSDP). Master + workers, 8 GPU/node.
+# Configure your cluster via env: MASTER_NODE (host), WORKER_NODES, MASTER_ADDR (host IB/high-speed IP),
+# and the NCCL_* / IFNAME settings below (cluster-specific — adjust to your fabric).
+# Run from an orchestrator node; the master is launched remotely over ssh (no local training process).
 # 사용: scripts/launch.sh <rdzv_id> <config_abspath> <outdir> <logdir> [extra train.py args...]
 set -u
-R=/NHNHOME/WORKSPACE/0526040027_A/OpenPath
+R="${OPENPATH_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}"
 RDZV="$1"; CFG="$2"; OUTDIR="$3"; LOGDIR="$4"; shift 4
 EXTRA="$*"
-HOST="TJ-2"
-EP="10.34.15.21:29500"           # TJ-2 IB IP (rendezvous rank0)
-WORKERS="TJ-3 TJ-4 TJ-5 TJ-6"
+HOST="${MASTER_NODE:-node1}"                  # master node hostname
+EP="${MASTER_ADDR:-10.0.0.1}:29500"           # master node IB/high-speed IP (rendezvous rank0)
+WORKERS="${WORKER_NODES:-node2 node3 node4 node5}"
+NNODES=$(( 1 + $(echo "$WORKERS" | wc -w) ))  # host + workers
 PYC="\$HOME/.cache/op_pyc_$RDZV"  # 노드-로컬. 원격 셸에서 확장되도록 escape
-ENV="WANDB_MODE=disabled NCCL_DEBUG=WARN NCCL_IB_HCA=mlx5_1,mlx5_2,mlx5_4,mlx5_9 \
-NCCL_SOCKET_IFNAME=bond0 GLOO_SOCKET_IFNAME=bond0 \
+# ★ NCCL/네트워크는 클러스터 특정 — 자신의 IB HCA/인터페이스로 교체:
+ENV="WANDB_MODE=disabled NCCL_DEBUG=WARN NCCL_IB_HCA=${NCCL_IB_HCA:-mlx5_0} \
+NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-eth0} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-eth0} \
 NCCL_IB_TIMEOUT=22 NCCL_IB_RETRY_CNT=13 NCCL_IB_QPS_PER_CONNECTION=4 \
 PYTHONDONTWRITEBYTECODE=1 PYTHONPYCACHEPREFIX=$PYC PYTHONPATH=$R/OpenPath"
-TR="venv/bin/torchrun --nnodes=5 --nproc-per-node=8 --rdzv-backend=c10d \
+TR="venv/bin/torchrun --nnodes=$NNODES --nproc-per-node=8 --rdzv-backend=c10d \
 --rdzv-endpoint=$EP --rdzv-id=$RDZV \
 OpenPath/dinov2/train/train.py --config-file $CFG --output-dir $OUTDIR $EXTRA"
 
 mkdir -p "$LOGDIR" "$OUTDIR"
 
-# 1) host(TJ-2) 원격 기동 — ★ssh를 &로 백그라운드(안 그러면 원격 setsid의 상속 fd를 물고 ssh가 반환 안 함=런처 행)
-echo "launching host on $HOST ..."
+# 1) launch master remotely — ★ssh를 &로 백그라운드(안 그러면 원격 setsid의 상속 fd를 물고 ssh가 반환 안 함=런처 행)
+echo "launching master on $HOST ..."
 ssh -n -o BatchMode=yes "$HOST" "mkdir -p $PYC; cd $R && setsid env $ENV $TR > $LOGDIR/${HOST}.log 2>&1 < /dev/null & echo ${HOST}-host-fired" &
 
 # 2) host torchrun가 store 바인딩할 시간(고정 대기). c10d는 워커가 재시도로 붙으므로 ss폴링 불필요.
-echo "waiting 30s for host store bind on $EP ..."
+echo "waiting 30s for master store bind on $EP ..."
 sleep 30
-echo "firing 4 workers"
+echo "firing workers"
 
-# 3) 워커(TJ-3~6) 병렬 기동
+# 3) launch workers in parallel
 for n in $WORKERS; do
   ssh -n -o BatchMode=yes "$n" "mkdir -p $PYC; cd $R && setsid env $ENV $TR > $LOGDIR/${n}.log 2>&1 < /dev/null & echo ${n}-fired" &
 done
 # ★ wait 대신 고정 sleep — bg ssh가 원격 setsid의 fd를 물어 반환 안 할 수 있어 wait는 영원히 멈춤(autoresume blocking 호출도 막힘).
 sleep 20
-echo "ALL launched: host $HOST + workers $WORKERS | rdzv=$RDZV | logs=$LOGDIR | out=$OUTDIR"
+echo "ALL launched: master $HOST + workers $WORKERS | rdzv=$RDZV | logs=$LOGDIR | out=$OUTDIR"

scripts/watch_eval.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
-# TJ-1 실시간 HEST eval watcher. run 학습이 뱉는 teacher_checkpoint를 감지→probe→곡선 append.
-# TJ-1 로컬 실행(venv_hest·GPU 로컬). 1개씩 순차(동시 HDF5 버그 회피, ~60min<체크포인트간격 ~84min라 페이스 유지).
+# HEST eval watcher (run on a node with a local GPU). 학습이 뱉는 teacher_checkpoint를 감지→probe→곡선 append.
+# 로컬 실행(venv_hest·GPU 로컬). 1개씩 순차(동시 HDF5 버그 회피, ~60min<체크포인트간격 ~84min라 페이스 유지).
 # per-exp embed 분리(/dev/shm/hest_emb_<N>), pkill 미포함(self-kill 회피).
 set -u
-R=/NHNHOME/WORKSPACE/0526040027_A/OpenPath
+R="${OPENPATH_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}"
 OUTDIR=$R/data/runs/openpath_run
 EVALDIR=$OUTDIR/eval
 RESDIR=$R/data/runs/hest_3way
@@ -12,12 +12,12 @@ BENCH=/dev/shm/hest_bench
 GPU=0
 mkdir -p "$RESDIR"
 
-# hest_bench를 TJ-1 /dev/shm에 확보(없으면 Lustre서 복사, 40G)
+# hest_bench를 로컬 /dev/shm에 확보(없으면 Lustre서 복사, 40G)
 if [ ! -d "$BENCH" ] || [ "$(ls "$BENCH" 2>/dev/null | wc -l)" -lt 5 ]; then
   echo "[watch] copying hest_bench -> $BENCH (40G, 수분) ..."
   rm -rf "$BENCH"; cp -r "$R/data/eva/hest_bench" "$BENCH"
 fi
-echo "[watch] START — watching $EVALDIR (TJ-1 GPU$GPU, 1-at-a-time) | curve=$CURVE"
+echo "[watch] START — watching $EVALDIR (GPU$GPU, 1-at-a-time) | curve=$CURVE"
 
 cd "$R"
 while true; do