Image-Text-to-Text
PEFT
Safetensors
gemma4
lora
qlora
ocr
indic
vision-language
drishti / code /run_train.sh
srivassid's picture
Gemma 4 12B Indic OCR/caption LoRA v1: adapter, card, results, code
9df0eb7 verified
Raw History Blame Contribute Delete
1.66 kB
#!/bin/bash
# Supervisor: remounts data, runs training, resumes from the newest checkpoint after crash/reboot.
# Launch: nohup /mnt/hdd/vlmin/run_train.sh >> /mnt/hdd/vlmin/runs/supervisor.log 2>&1 &
# Stops for good when runs/<RUN>.DONE exists. Safe to launch twice (flock).
CFG=${CFG:-configs/train_gemma4_12b.yaml}
RUN=${RUN:-gemma4-12b-indic-v2}
LIMIT=${LIMIT:-36000}
GPU=${GPU:-1,2}
BS=${BS:-2}
NGPU=$(echo $GPU | tr "," "\n" | wc -l)
cd /mnt/hdd/vlmin/VLM-IN || exit 1
DONE=/mnt/hdd/vlmin/runs/$RUN.DONE
LOCK=/mnt/hdd/vlmin/runs/$RUN.lock
exec 9>"$LOCK"; flock -n 9 || { echo "$(date) already running"; exit 0; }
[ -f "$DONE" ] && { echo "$(date) already finished"; exit 0; }
export PYTHONPATH=$PWD/src CUDA_VISIBLE_DEVICES=$GPU TOKENIZERS_PARALLELISM=false
for try in $(seq 1 50); do
if ! ls ~/data50/VLM-IN >/dev/null 2>&1; then
fusermount -uz ~/data50 2>/dev/null
mkdir -p ~/data50
sshfs -o ro,IdentityFile=$HOME/.ssh/id_to50,reconnect,ServerAliveInterval=15,cache=yes,kernel_cache,StrictHostKeyChecking=accept-new administrator@192.168.29.50:/F:/Data ~/data50
fi
if ls ~/data50/VLM-IN/processed/mix/train.jsonl >/dev/null 2>&1; then
echo "$(date) start attempt $try"
../venv/bin/torchrun --nproc_per_node=$NGPU --master_port=29511 -m vlmin.train --config $CFG --run-name $RUN --train-limit $LIMIT --set train.save_steps=50 --set train.per_device_train_batch_size=$BS --set train.gradient_accumulation_steps=$((16/NGPU/BS)) >> /mnt/hdd/vlmin/runs/$RUN.log 2>&1
rc=$?
echo "$(date) exit $rc"
[ $rc -eq 0 ] && { touch "$DONE"; exit 0; }
else
echo "$(date) data not reachable, retry"
fi
sleep 120
done