#!/usr/bin/env python3 """@file compare_music_tests.py @description A/B compare Moldovan test-song WAVs across the 3 music models. Reads /tmp/moldovan_tests/*.wav (MiniMax baseline + YuE2 + ACE-Step), reports ffprobe audio metrics per file, verifies all three exist, and prints the verdict template for the lyric-quality A/B. Usage: python3 scripts/compare_music_tests.py [--dir /tmp/moldovan_tests] """ from __future__ import annotations import argparse import json import subprocess import sys from pathlib import Path # Keys match run_music_tests.sh slugify() output exactly (dashes, cut at 30 chars). MODELS = { "minimax-music-3--full-song---v": "MiniMax Music 3 (baseline)", "yue2-3b--editable-scores--48kh": "YuE2-3B", "ace-step-v1-5-xl-turbo--commer": "ACE-Step v1.5 XL Turbo", } def probe(path: Path) -> dict | None: try: out = subprocess.run( [ "ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", str(path), ], capture_output=True, text=True, timeout=30, ).stdout d = json.loads(out) st = d.get("streams", [{}])[0] fmt = d.get("format", {}) return { "file": path.name, "seconds": round(float(fmt.get("duration", 0)), 1), "sample_rate": st.get("sample_rate"), "channels": st.get("channels"), "codec": st.get("codec_name"), "bytes": path.stat().st_size, } except Exception as exc: return {"file": path.name, "error": str(exc)} def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--dir", default="/tmp/moldovan_tests") args = ap.parse_args() base = Path(args.dir) rows = [] missing = [] for slug, label in MODELS.items(): p = base / f"{slug}_test.wav" if p.exists(): rows.append((label, probe(p))) else: missing.append((label, slug)) print(f"=== Moldovan test-song A/B ({base}) ===") for label, m in rows: if "error" in m: print(f"{label:28} ERROR: {m['error']}") continue print( f"{label:28} {m['seconds']:>6}s {m['sample_rate']}Hz " f"{m['channels']}ch {m['codec']} {m['bytes']//1024}KB" ) for label, slug in missing: print(f"{label:28} MISSING ({slug}_test.wav) - waiting on quota ladder") if len(rows) < len(MODELS): print(f"\n[{len(rows)}/{len(MODELS)} models present] Verdict blocked until the quota ladder renders the rest.") return 2 # Objective lyric-alignment probe: force Romanian and count expected words. # Uses faster-whisper when importable (scratch venv), else skips cleanly. try: from faster_whisper import WhisperModel # type: ignore wm = WhisperModel("small", device="cpu", compute_type="int8") expected = ["liniste", "ploua", "paine", "poiana"] print("\n=== Vocal lyric alignment (Whisper small, language=ro) ===") for slug, label in MODELS.items(): p = base / f"{slug}_test.wav" if not p.exists(): continue segs, _ = wm.transcribe(str(p), language="ro", beam_size=5) txt = " ".join(s.text.strip() for s in segs).lower() hits = sum(1 for w in expected if w in txt) print(f"{label:28} ro-match {hits}/4 | {txt[:150]}") except ImportError: print("\n(faster-whisper not importable; skipping vocal alignment probe)") # Lyric-quality judgement is human/LLM-eval; print the checklist template. print("\n=== Lyric-quality A/B checklist (Moldovan lines: 'Liniste, liniste, afara ploua...') ===") for line in [ "1. Pronunciation of Romanian diacritics (ă/â/ș/ț) by the vocal model", "2. Lyric-to-vocal alignment (are the Moldovan lines actually sung?)", "3. Instrumental authenticity (manele/balkan vibe vs generic EDM)", "4. Intelligibility at 15s length (hook formation)", "5. Artifacts/clipping (48kHz sources: YuE2, ACE-Step)", ]: print(f" {line}") print("\nPlay the three WAVs and score 1-5 per row; overall verdict = best score sum.") return 0 if __name__ == "__main__": sys.exit(main())