#!/bin/bash # Supervisor: remounts data, runs training, resumes from the newest checkpoint after crash/reboot. # Launch: nohup /mnt/hdd/vlmin/run_train.sh >> /mnt/hdd/vlmin/runs/supervisor.log 2>&1 & # Stops for good when runs/.DONE exists. Safe to launch twice (flock). CFG=${CFG:-configs/train_gemma4_12b.yaml} RUN=${RUN:-gemma4-12b-indic-v2} LIMIT=${LIMIT:-36000} GPU=${GPU:-1,2} BS=${BS:-2} NGPU=$(echo $GPU | tr "," "\n" | wc -l) cd /mnt/hdd/vlmin/VLM-IN || exit 1 DONE=/mnt/hdd/vlmin/runs/$RUN.DONE LOCK=/mnt/hdd/vlmin/runs/$RUN.lock exec 9>"$LOCK"; flock -n 9 || { echo "$(date) already running"; exit 0; } [ -f "$DONE" ] && { echo "$(date) already finished"; exit 0; } export PYTHONPATH=$PWD/src CUDA_VISIBLE_DEVICES=$GPU TOKENIZERS_PARALLELISM=false for try in $(seq 1 50); do if ! ls ~/data50/VLM-IN >/dev/null 2>&1; then fusermount -uz ~/data50 2>/dev/null mkdir -p ~/data50 sshfs -o ro,IdentityFile=$HOME/.ssh/id_to50,reconnect,ServerAliveInterval=15,cache=yes,kernel_cache,StrictHostKeyChecking=accept-new administrator@192.168.29.50:/F:/Data ~/data50 fi if ls ~/data50/VLM-IN/processed/mix/train.jsonl >/dev/null 2>&1; then echo "$(date) start attempt $try" ../venv/bin/torchrun --nproc_per_node=$NGPU --master_port=29511 -m vlmin.train --config $CFG --run-name $RUN --train-limit $LIMIT --set train.save_steps=50 --set train.per_device_train_batch_size=$BS --set train.gradient_accumulation_steps=$((16/NGPU/BS)) >> /mnt/hdd/vlmin/runs/$RUN.log 2>&1 rc=$? echo "$(date) exit $rc" [ $rc -eq 0 ] && { touch "$DONE"; exit 0; } else echo "$(date) data not reachable, retry" fi sleep 120 done