Image-Text-to-Text
PEFT
Safetensors
gemma4
lora
qlora
ocr
indic
vision-language
File size: 1,656 Bytes
9df0eb7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#!/bin/bash
# Supervisor: remounts data, runs training, resumes from the newest checkpoint after crash/reboot.
# Launch: nohup /mnt/hdd/vlmin/run_train.sh >> /mnt/hdd/vlmin/runs/supervisor.log 2>&1 &
# Stops for good when runs/<RUN>.DONE exists. Safe to launch twice (flock).
CFG=${CFG:-configs/train_gemma4_12b.yaml}
RUN=${RUN:-gemma4-12b-indic-v2}
LIMIT=${LIMIT:-36000}
GPU=${GPU:-1,2}
BS=${BS:-2}
NGPU=$(echo $GPU | tr "," "\n" | wc -l)
cd /mnt/hdd/vlmin/VLM-IN || exit 1
DONE=/mnt/hdd/vlmin/runs/$RUN.DONE
LOCK=/mnt/hdd/vlmin/runs/$RUN.lock
exec 9>"$LOCK"; flock -n 9 || { echo "$(date) already running"; exit 0; }
[ -f "$DONE" ] && { echo "$(date) already finished"; exit 0; }
export PYTHONPATH=$PWD/src CUDA_VISIBLE_DEVICES=$GPU TOKENIZERS_PARALLELISM=false
for try in $(seq 1 50); do
  if ! ls ~/data50/VLM-IN >/dev/null 2>&1; then
    fusermount -uz ~/data50 2>/dev/null
    mkdir -p ~/data50
    sshfs -o ro,IdentityFile=$HOME/.ssh/id_to50,reconnect,ServerAliveInterval=15,cache=yes,kernel_cache,StrictHostKeyChecking=accept-new administrator@192.168.29.50:/F:/Data ~/data50
  fi
  if ls ~/data50/VLM-IN/processed/mix/train.jsonl >/dev/null 2>&1; then
    echo "$(date) start attempt $try"
    ../venv/bin/torchrun --nproc_per_node=$NGPU --master_port=29511 -m vlmin.train --config $CFG --run-name $RUN --train-limit $LIMIT --set train.save_steps=50 --set train.per_device_train_batch_size=$BS --set train.gradient_accumulation_steps=$((16/NGPU/BS)) >> /mnt/hdd/vlmin/runs/$RUN.log 2>&1
    rc=$?
    echo "$(date) exit $rc"
    [ $rc -eq 0 ] && { touch "$DONE"; exit 0; }
  else
    echo "$(date) data not reachable, retry"
  fi
  sleep 120
done