flowmaster commited on
Commit
5fbb200
·
1 Parent(s): 946a961

test(music): add objective vocal-alignment probe to the A/B harness

Browse files

Force Romanian Whisper transcription of each rendered test song and count
expected Moldovan words. Turns the A/B from a listening impression into a
measurable signal: MiniMax 0/4 (vocalised as hums at 15s), YuE2-3B 2/4,
ACE-Step 1/4.

Files changed (1) hide show
  1. scripts/compare_music_tests.py +19 -0
scripts/compare_music_tests.py CHANGED
@@ -79,6 +79,25 @@ def main() -> int:
79
  print(f"\n[{len(rows)}/{len(MODELS)} models present] Verdict blocked until the quota ladder renders the rest.")
80
  return 2
81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  # Lyric-quality judgement is human/LLM-eval; print the checklist template.
83
  print("\n=== Lyric-quality A/B checklist (Moldovan lines: 'Liniste, liniste, afara ploua...') ===")
84
  for line in [
 
79
  print(f"\n[{len(rows)}/{len(MODELS)} models present] Verdict blocked until the quota ladder renders the rest.")
80
  return 2
81
 
82
+ # Objective lyric-alignment probe: force Romanian and count expected words.
83
+ # Uses faster-whisper when importable (scratch venv), else skips cleanly.
84
+ try:
85
+ from faster_whisper import WhisperModel # type: ignore
86
+
87
+ wm = WhisperModel("small", device="cpu", compute_type="int8")
88
+ expected = ["liniste", "ploua", "paine", "poiana"]
89
+ print("\n=== Vocal lyric alignment (Whisper small, language=ro) ===")
90
+ for slug, label in MODELS.items():
91
+ p = base / f"{slug}_test.wav"
92
+ if not p.exists():
93
+ continue
94
+ segs, _ = wm.transcribe(str(p), language="ro", beam_size=5)
95
+ txt = " ".join(s.text.strip() for s in segs).lower()
96
+ hits = sum(1 for w in expected if w in txt)
97
+ print(f"{label:28} ro-match {hits}/4 | {txt[:150]}")
98
+ except ImportError:
99
+ print("\n(faster-whisper not importable; skipping vocal alignment probe)")
100
+
101
  # Lyric-quality judgement is human/LLM-eval; print the checklist template.
102
  print("\n=== Lyric-quality A/B checklist (Moldovan lines: 'Liniste, liniste, afara ploua...') ===")
103
  for line in [