File size: 2,615 Bytes
5c2beba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | #!/bin/bash
# Usage: ckpt_eval.sh <step_XXXXXXXXX.ckpt> [more ckpts...]
# Pauses training, evals baseline (once) + given checkpoints, resumes training.
set -uo pipefail
exec > /opt/work/ckpt_eval2.log 2>&1
CKPTDIR=/opt/work/fish-speech/results/s2pro_egy_lora/checkpoints
NV=/opt/work/asr_env/.venv/lib/python3.12/site-packages/nvidia
PY=/opt/work/fish-speech/.venv/bin/python
echo "=== ckpt_eval $(date) args: $* ==="
echo "== pausing training =="
tmux kill-session -t train 2>/dev/null
pkill -f "fish_speech/train.py" 2>/dev/null
for i in $(seq 1 30); do pgrep -f "fish_speech/train.py" >/dev/null || break; sleep 2; done
sleep 3
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
cd /opt/work/fish-speech
if [ ! -f /opt/work/eval/baseline_true/asr_report_soniox.json ]; then
echo "== baseline (zero-LoRA) synth =="
$PY /opt/work/scripts/synth_eval.py --outdir /opt/work/eval/baseline_true
echo "== baseline ASR =="
source /opt/work/scripts/soniox.env && \
/opt/work/asr_env/.venv/bin/python /opt/work/scripts/asr_eval_soniox.py --wavdir /opt/work/eval/baseline_true
fi
for CKPT in "$@"; do
NAME=$(echo "$CKPT" | grep -oE "step_[0-9]+" | sed "s/step_0*/step/")
echo "== $NAME synth ($CKPT) =="
$PY /opt/work/scripts/synth_eval.py --outdir /opt/work/eval/$NAME \
--lora-ckpt "$CKPTDIR/$CKPT" --lora-config r_32_egy
echo "== $NAME ASR =="
source /opt/work/scripts/soniox.env && \
/opt/work/asr_env/.venv/bin/python /opt/work/scripts/asr_eval_soniox.py --wavdir /opt/work/eval/$NAME
done
echo "== resuming training (auto-resume from latest ckpt) =="
tmux new-session -d -s train "cd /opt/work/fish-speech && export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TOKENIZERS_PARALLELISM=false && .venv/bin/python fish_speech/train.py --config-name text2semantic_finetune_egy +lora@model.model.lora_config=r_32_egy 2>&1 | tee /opt/work/train.log"
sleep 3; tmux has-session -t train && echo TRAIN_RESUMED || echo TMUX_FAILED
echo "== COMPARISON =="
$PY - << "PYEOF"
import json
from pathlib import Path
for d in sorted(Path("/opt/work/eval").iterdir()):
rep = d / "asr_report_soniox.json"
if rep.exists():
s = json.loads(rep.read_text())["summary"]
tj = d / "timing.jsonl"
rtf = "-"
if tj.exists():
rows = [json.loads(l) for l in tj.read_text().splitlines() if l.strip()]
ok = [r for r in rows if "rtf" in r]
if ok:
rtf = round(sum(r["rtf"] for r in ok) / len(ok), 2)
print(f"{d.name:16s} WER={s['mean_wer']:.3f} CER={s['mean_cer']:.3f} RTF={rtf}")
PYEOF
echo "CKPT_EVAL_DONE"
|