| #!/bin/bash |
| |
| . ~/asr/env.sh; cd ~/asr; L=/scratch/logs; E="PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" |
| while pgrep -f "runs/A_real " >/dev/null || pgrep -f "runs/A_real$" >/dev/null || pgrep -f "train_asr.py --run /scratch/asr/runs/A_real" >/dev/null; do sleep 30; done |
| echo "A finished $(date -u)"; grep -E "TRAIN_DONE|Error" $L/train_A.log | tail -2 |
| nohup ~/venv/bin/python -u synth_cs.py 4 5 > $L/synth_4.log 2>&1 & |
| ~/venv_omni/bin/python -u eval_asr.py --tag v1_stock --card omniASR_CTC_1B --sets duplex,realcs > $L/eval_v1.log 2>&1; echo "eval v1 $?" |
| ~/venv_omni/bin/python -u eval_asr.py --tag v2_stock --card omniASR_CTC_1B_v2 --sets duplex,realcs > $L/eval_v2.log 2>&1; echo "eval v2 $?" |
| ~/venv_omni/bin/python -u eval_asr.py --tag A_real --ckpt /scratch/asr/runs/A_real/best.pt --sets duplex,realcs > $L/eval_A.log 2>&1; echo "eval A $?" |
| DL=$(date -u -d "2026-10-24 18:40" +%s) |
| while [ $(date -u +%s) -lt $DL ]; do |
| n=$(grep -l SYNTH_DONE $L/synth_[0-4].log 2>/dev/null | wc -l); [ "$n" -ge 5 ] && break; sleep 30; done |
| pkill -f synth_cs.py; sleep 5; echo "synth stopped $(date -u)" |
| python3 build_synth.py merge |
| ~/venv_omni/bin/python -u eval_asr.py --hyp_only /scratch/asr/synth/all.jsonl > $L/synth_hyp.log 2>&1; echo "hyp $?" |
| python3 build_synth.py filter |
| env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True MEM_FRAC=0.9 ~/venv_omni/bin/python -u train_asr.py --run /scratch/asr/runs/B_synth --steps 4000 --batch_sec 160 --max_sec 18 --val_every 500 \ |
| --synth /scratch/asr/synth/manifest.jsonl --synth_frac 0.12 > $L/train_B.log 2>&1; echo "train B $?" |
| ~/venv_omni/bin/python -u eval_asr.py --tag B_synth --ckpt /scratch/asr/runs/B_synth/best.pt > $L/eval_B.log 2>&1; echo "eval B $?" |
| for t in "v1_stock --card omniASR_CTC_1B" "v2_stock --card omniASR_CTC_1B_v2" "A_real --ckpt /scratch/asr/runs/A_real/best.pt"; do |
| set -- $t; ~/venv_omni/bin/python -u eval_asr.py --tag ${1}_st $2 $3 --sets synthtest,ace > $L/eval_st_$1.log 2>&1; echo "eval st $1 $?"; done |
| echo PIPE_DONE $(date -u) |
|
|
