scripts/run_eval.py

#!/usr/bin/env python3
"""
CLI entry point for local / CI evaluation (same backends as the Gradio Space).

Usage:
  python scripts/run_eval.py --model openai/whisper-tiny --family auto
  python scripts/run_eval.py --model org/model --family transformers_ctc
"""

from __future__ import annotations

import argparse
import json
import os
import sys

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if ROOT not in sys.path:
    sys.path.insert(0, ROOT)

from backends.registry import FAMILY_IDS, default_family_id  # noqa: E402
from evaluation.orchestrator import run_evaluation  # noqa: E402


def main() -> None:
    parser = argparse.ArgumentParser(description="FFASR leaderboard evaluation (offline)")
    parser.add_argument("--model", required=True, help="Hugging Face model id, e.g. openai/whisper-tiny")
    parser.add_argument(
        "--family",
        default=default_family_id(),
        choices=FAMILY_IDS,
        help="Inference backend (must match the model architecture).",
    )
    parser.add_argument("--json", action="store_true", help="Print results as JSON")
    args = parser.parse_args()

    out = run_evaluation(args.model, args.family)
    if args.json:
        print(json.dumps(out, indent=2))
    else:
        print("model_id:", out["model_id"])
        print("eval_family:", out["eval_family"])
        print("wer_clean:", out["wer_clean"])
        print("wer_noisy:", out["wer_noisy"])
        print("wer_reverberant:", out["wer_reverberant"])
        print("wer_real:", out.get("wer_real"))
        print("wer_difficult:", out.get("wer_difficult"))
        print("num_samples:", out["num_samples"])


if __name__ == "__main__":
    main()