Spaces:
Running on Zero
Running on Zero
| #!/usr/bin/env python3 | |
| """@file compare_music_tests.py | |
| @description A/B compare Moldovan test-song WAVs across the 3 music models. | |
| Reads /tmp/moldovan_tests/*.wav (MiniMax baseline + YuE2 + ACE-Step), | |
| reports ffprobe audio metrics per file, verifies all three exist, | |
| and prints the verdict template for the lyric-quality A/B. | |
| Usage: | |
| python3 scripts/compare_music_tests.py [--dir /tmp/moldovan_tests] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| # Keys match run_music_tests.sh slugify() output exactly (dashes, cut at 30 chars). | |
| MODELS = { | |
| "minimax-music-3--full-song---v": "MiniMax Music 3 (baseline)", | |
| "yue2-3b--editable-scores--48kh": "YuE2-3B", | |
| "ace-step-v1-5-xl-turbo--commer": "ACE-Step v1.5 XL Turbo", | |
| } | |
| def probe(path: Path) -> dict | None: | |
| try: | |
| out = subprocess.run( | |
| [ | |
| "ffprobe", "-v", "quiet", "-print_format", "json", | |
| "-show_format", "-show_streams", str(path), | |
| ], | |
| capture_output=True, text=True, timeout=30, | |
| ).stdout | |
| d = json.loads(out) | |
| st = d.get("streams", [{}])[0] | |
| fmt = d.get("format", {}) | |
| return { | |
| "file": path.name, | |
| "seconds": round(float(fmt.get("duration", 0)), 1), | |
| "sample_rate": st.get("sample_rate"), | |
| "channels": st.get("channels"), | |
| "codec": st.get("codec_name"), | |
| "bytes": path.stat().st_size, | |
| } | |
| except Exception as exc: | |
| return {"file": path.name, "error": str(exc)} | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--dir", default="/tmp/moldovan_tests") | |
| args = ap.parse_args() | |
| base = Path(args.dir) | |
| rows = [] | |
| missing = [] | |
| for slug, label in MODELS.items(): | |
| p = base / f"{slug}_test.wav" | |
| if p.exists(): | |
| rows.append((label, probe(p))) | |
| else: | |
| missing.append((label, slug)) | |
| print(f"=== Moldovan test-song A/B ({base}) ===") | |
| for label, m in rows: | |
| if "error" in m: | |
| print(f"{label:28} ERROR: {m['error']}") | |
| continue | |
| print( | |
| f"{label:28} {m['seconds']:>6}s {m['sample_rate']}Hz " | |
| f"{m['channels']}ch {m['codec']} {m['bytes']//1024}KB" | |
| ) | |
| for label, slug in missing: | |
| print(f"{label:28} MISSING ({slug}_test.wav) - waiting on quota ladder") | |
| if len(rows) < len(MODELS): | |
| print(f"\n[{len(rows)}/{len(MODELS)} models present] Verdict blocked until the quota ladder renders the rest.") | |
| return 2 | |
| # Objective lyric-alignment probe: force Romanian and count expected words. | |
| # Uses faster-whisper when importable (scratch venv), else skips cleanly. | |
| try: | |
| from faster_whisper import WhisperModel # type: ignore | |
| wm = WhisperModel("small", device="cpu", compute_type="int8") | |
| expected = ["liniste", "ploua", "paine", "poiana"] | |
| print("\n=== Vocal lyric alignment (Whisper small, language=ro) ===") | |
| for slug, label in MODELS.items(): | |
| p = base / f"{slug}_test.wav" | |
| if not p.exists(): | |
| continue | |
| segs, _ = wm.transcribe(str(p), language="ro", beam_size=5) | |
| txt = " ".join(s.text.strip() for s in segs).lower() | |
| hits = sum(1 for w in expected if w in txt) | |
| print(f"{label:28} ro-match {hits}/4 | {txt[:150]}") | |
| except ImportError: | |
| print("\n(faster-whisper not importable; skipping vocal alignment probe)") | |
| # Lyric-quality judgement is human/LLM-eval; print the checklist template. | |
| print("\n=== Lyric-quality A/B checklist (Moldovan lines: 'Liniste, liniste, afara ploua...') ===") | |
| for line in [ | |
| "1. Pronunciation of Romanian diacritics (ă/â/ș/ț) by the vocal model", | |
| "2. Lyric-to-vocal alignment (are the Moldovan lines actually sung?)", | |
| "3. Instrumental authenticity (manele/balkan vibe vs generic EDM)", | |
| "4. Intelligibility at 15s length (hook formation)", | |
| "5. Artifacts/clipping (48kHz sources: YuE2, ACE-Step)", | |
| ]: | |
| print(f" {line}") | |
| print("\nPlay the three WAVs and score 1-5 per row; overall verdict = best score sum.") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) |