File size: 1,631 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | #!/bin/bash
set -uo pipefail
export HF_HOME=/scratch/hf PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
/root/venv/bin/python /root/train_wbert.py --lang lug --out /scratch/runs/lug_s2 \
--init /scratch/runs/lug_s1/best \
--train /scratch/prep/manifests/waxal_lug_train.jsonl \
--eval /scratch/prep/manifests/waxal_lug_validation.jsonl \
--lr 1e-5 --epochs 8 --bs 12 --grad_accum 3 --num_workers 18 --eval_steps 100 > /root/lug_s2.log 2>&1
if ! grep -q "TRAIN_DONE" /root/lug_s2.log; then echo "ERREUR: stage 2 en echec" >> /root/chain_lug.log; exit 1; fi
echo "=== stage 2 fini ($(date +%H:%M:%S)) ===" >> /root/chain_lug.log
grep -A9 "EVAL FINALE" /root/lug_s2.log | head -10 >> /root/chain_lug.log
mkdir -p /root/models /scratch/submissions
cp -r /scratch/runs/lug_s1/best /root/models/lug_s1_best 2>/dev/null || true
cp -r /scratch/runs/lug_s2/best /root/models/lug_s2_best
cp /scratch/runs/lug_s1/final_metrics.json /root/models/lug_s1_metrics.json 2>/dev/null || true
cp /scratch/runs/lug_s2/final_metrics.json /root/models/lug_s2_metrics.json 2>/dev/null || true
echo "=== checkpoints archives sur disque persistant ===" >> /root/chain_lug.log
/root/venv/bin/python /root/infer_submit.py --out /scratch/submissions/sub_probe_lug.csv \
--model lug=/scratch/runs/lug_s2/best --score_val > /root/submit_probe.log 2>&1
if ! grep -q "SUBMIT_DONE" /root/submit_probe.log; then echo "ERREUR: soumission en echec" >> /root/chain_lug.log; exit 1; fi
grep -E "VAL lug|SOUMISSION" /root/submit_probe.log >> /root/chain_lug.log
cp /scratch/submissions/sub_probe_lug.csv /root/models/
echo "CHAIN_DONE" >> /root/chain_lug.log
|