| #!/bin/bash |
| set -uo pipefail |
| export HF_HOME=/scratch/hf PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True |
| /root/venv/bin/python /root/train_wbert.py --lang lug --out /scratch/runs/lug_s2 \ |
| --init /scratch/runs/lug_s1/best \ |
| --train /scratch/prep/manifests/waxal_lug_train.jsonl \ |
| --eval /scratch/prep/manifests/waxal_lug_validation.jsonl \ |
| --lr 1e-5 --epochs 8 --bs 12 --grad_accum 3 --num_workers 18 --eval_steps 100 > /root/lug_s2.log 2>&1 |
| if ! grep -q "TRAIN_DONE" /root/lug_s2.log; then echo "ERREUR: stage 2 en echec" >> /root/chain_lug.log; exit 1; fi |
| echo "=== stage 2 fini ($(date +%H:%M:%S)) ===" >> /root/chain_lug.log |
| grep -A9 "EVAL FINALE" /root/lug_s2.log | head -10 >> /root/chain_lug.log |
| mkdir -p /root/models /scratch/submissions |
| cp -r /scratch/runs/lug_s1/best /root/models/lug_s1_best 2>/dev/null || true |
| cp -r /scratch/runs/lug_s2/best /root/models/lug_s2_best |
| cp /scratch/runs/lug_s1/final_metrics.json /root/models/lug_s1_metrics.json 2>/dev/null || true |
| cp /scratch/runs/lug_s2/final_metrics.json /root/models/lug_s2_metrics.json 2>/dev/null || true |
| echo "=== checkpoints archives sur disque persistant ===" >> /root/chain_lug.log |
| /root/venv/bin/python /root/infer_submit.py --out /scratch/submissions/sub_probe_lug.csv \ |
| --model lug=/scratch/runs/lug_s2/best --score_val > /root/submit_probe.log 2>&1 |
| if ! grep -q "SUBMIT_DONE" /root/submit_probe.log; then echo "ERREUR: soumission en echec" >> /root/chain_lug.log; exit 1; fi |
| grep -E "VAL lug|SOUMISSION" /root/submit_probe.log >> /root/chain_lug.log |
| cp /scratch/submissions/sub_probe_lug.csv /root/models/ |
| echo "CHAIN_DONE" >> /root/chain_lug.log |
|
|
