Spaces:
Running on Zero
Running on Zero
flowmaster commited on
Commit ·
5fbb200
1
Parent(s): 946a961
test(music): add objective vocal-alignment probe to the A/B harness
Browse filesForce Romanian Whisper transcription of each rendered test song and count
expected Moldovan words. Turns the A/B from a listening impression into a
measurable signal: MiniMax 0/4 (vocalised as hums at 15s), YuE2-3B 2/4,
ACE-Step 1/4.
scripts/compare_music_tests.py
CHANGED
|
@@ -79,6 +79,25 @@ def main() -> int:
|
|
| 79 |
print(f"\n[{len(rows)}/{len(MODELS)} models present] Verdict blocked until the quota ladder renders the rest.")
|
| 80 |
return 2
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
# Lyric-quality judgement is human/LLM-eval; print the checklist template.
|
| 83 |
print("\n=== Lyric-quality A/B checklist (Moldovan lines: 'Liniste, liniste, afara ploua...') ===")
|
| 84 |
for line in [
|
|
|
|
| 79 |
print(f"\n[{len(rows)}/{len(MODELS)} models present] Verdict blocked until the quota ladder renders the rest.")
|
| 80 |
return 2
|
| 81 |
|
| 82 |
+
# Objective lyric-alignment probe: force Romanian and count expected words.
|
| 83 |
+
# Uses faster-whisper when importable (scratch venv), else skips cleanly.
|
| 84 |
+
try:
|
| 85 |
+
from faster_whisper import WhisperModel # type: ignore
|
| 86 |
+
|
| 87 |
+
wm = WhisperModel("small", device="cpu", compute_type="int8")
|
| 88 |
+
expected = ["liniste", "ploua", "paine", "poiana"]
|
| 89 |
+
print("\n=== Vocal lyric alignment (Whisper small, language=ro) ===")
|
| 90 |
+
for slug, label in MODELS.items():
|
| 91 |
+
p = base / f"{slug}_test.wav"
|
| 92 |
+
if not p.exists():
|
| 93 |
+
continue
|
| 94 |
+
segs, _ = wm.transcribe(str(p), language="ro", beam_size=5)
|
| 95 |
+
txt = " ".join(s.text.strip() for s in segs).lower()
|
| 96 |
+
hits = sum(1 for w in expected if w in txt)
|
| 97 |
+
print(f"{label:28} ro-match {hits}/4 | {txt[:150]}")
|
| 98 |
+
except ImportError:
|
| 99 |
+
print("\n(faster-whisper not importable; skipping vocal alignment probe)")
|
| 100 |
+
|
| 101 |
# Lyric-quality judgement is human/LLM-eval; print the checklist template.
|
| 102 |
print("\n=== Lyric-quality A/B checklist (Moldovan lines: 'Liniste, liniste, afara ploua...') ===")
|
| 103 |
for line in [
|