Add Multi-Model API For Transcription

.gitignore

@@ -1 +1,4 @@
 *.mp3
+
+__pycache__/
+*.pyc

README.md

@@ -9,7 +9,48 @@ python_version: '3.12'
 app_file: app.py
 pinned: false
 license: mit
-short_description: Transcribe files with speaker diarization
+short_description: Multi-model ASR benchmarking with word-level timestamps
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This Space is optimized for API usage and benchmarking on ZeroGPU.
+
+Supported models (word-level timestamp capable):
+- Whisper Large V3
+- Whisper Large V3 Turbo
+- Whisper.cpp (large)
+- Whisper faster (large)
+- NVIDIA Parakeet v3
+
+Omitted:
+- IBM Granite Speech 3.3 8B (no stable, documented word-level timestamp output in standard inference APIs)
+
+Every transcription response returns:
+- raw model output object
+- `zerogpu_timing.gpu_window_seconds`
+- `zerogpu_timing.inference_seconds`
+
+Benchmark response (`/benchmark_all_models`) returns:
+- one item per supported model with `status` (`ok` or `error`)
+- each successful model's full raw output + timing
+- benchmark-level wall clock summary and speed leaderboard
+
+Whisper.cpp notes:
+- Requires a whisper.cpp binary and a model file.
+- Configure with env vars:
+  - `WHISPER_CPP_BIN` (default: `whisper-cli`)
+  - `WHISPER_CPP_MODEL_LARGE` (path to ggml model)
+
+API endpoints:
+- `/transcribe_selected`
+- `/benchmark_all_models`
+- `/transcribe_whisper_large_v3`
+- `/transcribe_whisper_large_v3_turbo`
+- `/transcribe_whisper_cpp_large`
+- `/transcribe_whisper_faster_large`
+- `/transcribe_parakeet_v3`
+
+Code structure:
+- `app.py`: Gradio wiring and API routes
+- `src/transcription_service.py`: dispatch + benchmark orchestration
+- `src/utils.py`: shared JSON/serialization helpers
+- `src/models/`: model-specific backend implementations

app.py

@@ -1,50 +1,269 @@
+import gradio as gr
 import spaces
-import torch
 
-import gradio as gr
-from transformers import pipeline
+from src.constants import (
+    FILE_LIMIT_MB,
+    OMITTED_MODELS,
+    PARAKEET_V3,
+    SUPPORTED_MODELS,
+    WHISPER_CPP_LARGE,
+    WHISPER_FASTER_LARGE,
+    WHISPER_LARGE_V3,
+    WHISPER_LARGE_V3_TURBO,
+)
+from src.transcription_service import benchmark_all_models, dispatch_transcription
 
 
-MODEL_NAME = "openai/whisper-large-v3"
-BATCH_SIZE = 8
-FILE_LIMIT_MB = 1000
+@spaces.GPU
+def transcribe_selected_model(
+    audio_file,
+    model_label,
+    task,
+    language,
+    initial_prompt,
+    postprocess_prompt,
+    model_options_json,
+):
+    return dispatch_transcription(
+        audio_file,
+        model_label,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    )
 
-device = 0 if torch.cuda.is_available() else "cpu"
 
-pipe = pipeline(
-    task="automatic-speech-recognition",
-    model=MODEL_NAME,
-    chunk_length_s=30,
-    device=device,
-)
+@spaces.GPU
+def transcribe_whisper_large_v3(
+    audio_file,
+    task,
+    language,
+    initial_prompt,
+    postprocess_prompt,
+    model_options_json,
+):
+    return dispatch_transcription(
+        audio_file,
+        WHISPER_LARGE_V3,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    )
 
 
 @spaces.GPU
-def transcribe(inputs, task):
-    if inputs is None:
-        raise gr.Error("No audio file submitted! Please upload or record an audio file before submitting your request.")
+def transcribe_whisper_large_v3_turbo(
+    audio_file,
+    task,
+    language,
+    initial_prompt,
+    postprocess_prompt,
+    model_options_json,
+):
+    return dispatch_transcription(
+        audio_file,
+        WHISPER_LARGE_V3_TURBO,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    )
 
-    text = pipe(inputs, batch_size=BATCH_SIZE, generate_kwargs={"task": task}, return_timestamps=True)
-    return text
 
+@spaces.GPU
+def transcribe_whisper_cpp_large(
+    audio_file,
+    task,
+    language,
+    initial_prompt,
+    postprocess_prompt,
+    model_options_json,
+):
+    return dispatch_transcription(
+        audio_file,
+        WHISPER_CPP_LARGE,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    )
+
+
+@spaces.GPU
+def transcribe_whisper_faster_large(
+    audio_file,
+    task,
+    language,
+    initial_prompt,
+    postprocess_prompt,
+    model_options_json,
+):
+    return dispatch_transcription(
+        audio_file,
+        WHISPER_FASTER_LARGE,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    )
+
+
+@spaces.GPU
+def transcribe_parakeet_v3(
+    audio_file,
+    task,
+    language,
+    initial_prompt,
+    postprocess_prompt,
+    model_options_json,
+):
+    return dispatch_transcription(
+        audio_file,
+        PARAKEET_V3,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    )
+
+
+@spaces.GPU
+def benchmark_models(
+    audio_file,
+    task,
+    language,
+    initial_prompt,
+    postprocess_prompt,
+    model_options_json,
+):
+    return benchmark_all_models(
+        audio_file,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    )
 
-demo = gr.Blocks()
 
-with demo:
-    gr.Interface(
-        fn=transcribe,
-        inputs=[
-            gr.Audio(sources="upload", type="filepath", label="Audio file"),
-            gr.Radio(["transcribe", "translate"], label="Task", value="transcribe"),
-        ],
-        outputs=gr.JSON(label="transcription"),
-        title="Whisper Large V3: Transcribe Audio",
-        description=(
-            "Transcribe long-form microphone or audio inputs with the click of a button! Demo uses the"
-            f" checkpoint [{MODEL_NAME}](https://huggingface.co/{MODEL_NAME}) and 🤗 Transformers to transcribe audio files"
-            " of arbitrary length."
-        ),
+with gr.Blocks(theme=gr.themes.Ocean(), title="Multi-model ASR benchmark (ZeroGPU)") as demo:
+    gr.Markdown(
+        "# Multi-model transcription benchmark (ZeroGPU)\n"
+        "API-first design with one endpoint per model and full raw outputs (including word-level timestamps)."
     )
 
+    with gr.Row():
+        audio_file = gr.Audio(
+            sources=["upload"],
+            type="filepath",
+            label="Audio file",
+            max_length=FILE_LIMIT_MB,
+        )
+
+    with gr.Row():
+        model_label = gr.Dropdown(
+            choices=SUPPORTED_MODELS,
+            value=WHISPER_LARGE_V3,
+            label="Model",
+        )
+        task = gr.Radio(
+            choices=["transcribe", "translate"],
+            value="transcribe",
+            label="Task",
+        )
+
+    with gr.Row():
+        language = gr.Textbox(label="Language code (optional)", placeholder="e.g. en")
+        initial_prompt = gr.Textbox(label="Initial prompt (optional)")
+
+    postprocess_prompt = gr.Textbox(
+        label="Post-processing prompt/instruction (optional, recorded in output metadata)",
+        lines=2,
+    )
+
+    model_options_json = gr.Textbox(
+        label="Model options JSON (optional)",
+        placeholder='{"beam_size": 5, "temperature": 0.0, "vad_filter": true}',
+        lines=3,
+    )
+
+    run_btn = gr.Button("Run selected model")
+    benchmark_btn = gr.Button("Benchmark all supported models")
+
+    output = gr.JSON(label="Raw transcription output + timing")
+
+    shared_inputs = [
+        audio_file,
+        task,
+        language,
+        initial_prompt,
+        postprocess_prompt,
+        model_options_json,
+    ]
+
+    run_btn.click(
+        fn=transcribe_selected_model,
+        inputs=[audio_file, model_label, *shared_inputs[1:]],
+        outputs=output,
+        api_name="transcribe_selected",
+    )
+
+    benchmark_btn.click(
+        fn=benchmark_models,
+        inputs=shared_inputs,
+        outputs=output,
+        api_name="benchmark_all_models",
+    )
+
+    # Hidden controls used only to expose dedicated API routes per model.
+    with gr.Row(visible=False):
+        api_btn_wlv3 = gr.Button("transcribe_whisper_large_v3")
+        api_btn_wlv3t = gr.Button("transcribe_whisper_large_v3_turbo")
+        api_btn_wcpp = gr.Button("transcribe_whisper_cpp_large")
+        api_btn_fw = gr.Button("transcribe_whisper_faster_large")
+        api_btn_parakeet = gr.Button("transcribe_parakeet_v3")
+
+    api_btn_wlv3.click(
+        fn=transcribe_whisper_large_v3,
+        inputs=shared_inputs,
+        outputs=output,
+        api_name="transcribe_whisper_large_v3",
+    )
+    api_btn_wlv3t.click(
+        fn=transcribe_whisper_large_v3_turbo,
+        inputs=shared_inputs,
+        outputs=output,
+        api_name="transcribe_whisper_large_v3_turbo",
+    )
+    api_btn_wcpp.click(
+        fn=transcribe_whisper_cpp_large,
+        inputs=shared_inputs,
+        outputs=output,
+        api_name="transcribe_whisper_cpp_large",
+    )
+    api_btn_fw.click(
+        fn=transcribe_whisper_faster_large,
+        inputs=shared_inputs,
+        outputs=output,
+        api_name="transcribe_whisper_faster_large",
+    )
+    api_btn_parakeet.click(
+        fn=transcribe_parakeet_v3,
+        inputs=shared_inputs,
+        outputs=output,
+        api_name="transcribe_parakeet_v3",
+    )
+
+    omitted = "\n".join([f"- {k}: {v}" for k, v in OMITTED_MODELS.items()])
+    gr.Markdown(f"## Omitted models\n{omitted}")
+
 
-demo.queue().launch(theme=gr.themes.Ocean(), ssr_mode=False)
+demo.queue().launch(ssr_mode=False)

requirements.txt

@@ -1,2 +1,5 @@
-transformers
-
+transformers>=4.46.0
+accelerate>=1.1.0
+torch>=2.3.0
+faster-whisper>=1.1.0
+nemo_toolkit[asr]>=2.0.0

src/__init__.py

@@ -0,0 +1 @@
+# Package marker.

src/constants.py

@@ -0,0 +1,29 @@
+BATCH_SIZE = 8
+FILE_LIMIT_MB = 1000
+
+WHISPER_LARGE_V3 = "Whisper Large V3"
+WHISPER_LARGE_V3_TURBO = "Whisper Large V3 Turbo"
+WHISPER_CPP_LARGE = "Whisper.cpp (large)"
+WHISPER_FASTER_LARGE = "Whisper faster (large)"
+PARAKEET_V3 = "NVIDIA Parakeet v3"
+
+SUPPORTED_MODELS = [
+    WHISPER_LARGE_V3,
+    WHISPER_LARGE_V3_TURBO,
+    WHISPER_CPP_LARGE,
+    WHISPER_FASTER_LARGE,
+    PARAKEET_V3,
+]
+
+OMITTED_MODELS = {
+    "IBM Granite Speech 3.3 8B": (
+        "Omitted because a stable, documented word-level timestamp interface is not available "
+        "in standard inference usage."
+    )
+}
+
+MODEL_IDS = {
+    WHISPER_LARGE_V3: "openai/whisper-large-v3",
+    WHISPER_LARGE_V3_TURBO: "openai/whisper-large-v3-turbo",
+    PARAKEET_V3: "nvidia/parakeet-tdt-0.6b-v3",
+}

src/models/__init__.py

@@ -0,0 +1 @@
+# Model backend package.

src/models/faster_whisper_model.py

@@ -0,0 +1,98 @@
+import time
+from typing import Any
+
+import gradio as gr
+import torch
+
+from src.utils import serialize
+
+_FASTER_WHISPER_MODELS: dict[str, Any] = {}
+
+
+def _get_faster_whisper_model(model_options: dict[str, Any]):
+    model_size = model_options.get("model_size", "large-v3")
+    compute_type = model_options.get(
+        "compute_type",
+        "float16" if torch.cuda.is_available() else "int8",
+    )
+    cache_key = f"{model_size}:{compute_type}"
+    if cache_key in _FASTER_WHISPER_MODELS:
+        return _FASTER_WHISPER_MODELS[cache_key], model_size, compute_type
+
+    try:
+        from faster_whisper import WhisperModel
+    except Exception as exc:
+        raise gr.Error(
+            "faster-whisper backend requested but package is missing. "
+            "Add faster-whisper to requirements.txt"
+        ) from exc
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = WhisperModel(model_size, device=device, compute_type=compute_type)
+    _FASTER_WHISPER_MODELS[cache_key] = model
+    return model, model_size, compute_type
+
+
+def run_faster_whisper(
+    audio_file: str,
+    task: str,
+    language: str,
+    initial_prompt: str,
+    model_options: dict[str, Any],
+) -> dict[str, Any]:
+    model, model_size, compute_type = _get_faster_whisper_model(model_options)
+    beam_size = int(model_options.get("beam_size", 5))
+    temperature = float(model_options.get("temperature", 0.0))
+    vad_filter = bool(model_options.get("vad_filter", True))
+
+    infer_start = time.perf_counter()
+    segments, info = model.transcribe(
+        audio_file,
+        task=task,
+        language=language or None,
+        initial_prompt=initial_prompt or None,
+        word_timestamps=True,
+        beam_size=beam_size,
+        temperature=temperature,
+        vad_filter=vad_filter,
+    )
+    segments_list = list(segments)
+    infer_end = time.perf_counter()
+
+    raw_output = {
+        "info": serialize(info),
+        "segments": [
+            {
+                "id": seg.id,
+                "seek": seg.seek,
+                "start": seg.start,
+                "end": seg.end,
+                "text": seg.text,
+                "tokens": list(seg.tokens) if seg.tokens is not None else None,
+                "avg_logprob": seg.avg_logprob,
+                "compression_ratio": seg.compression_ratio,
+                "no_speech_prob": seg.no_speech_prob,
+                "words": [
+                    {
+                        "start": w.start,
+                        "end": w.end,
+                        "word": w.word,
+                        "probability": w.probability,
+                    }
+                    for w in (seg.words or [])
+                ],
+            }
+            for seg in segments_list
+        ],
+        "runtime": {
+            "model_size": model_size,
+            "compute_type": compute_type,
+        },
+    }
+
+    return {
+        "raw_output": serialize(raw_output),
+        "timing": {
+            "inference_seconds": round(infer_end - infer_start, 4),
+        },
+    }

src/models/parakeet_model.py

@@ -0,0 +1,56 @@
+import time
+
+import gradio as gr
+import torch
+
+from src.constants import MODEL_IDS, PARAKEET_V3
+from src.utils import serialize
+
+_PARAKEET_MODEL = None
+
+
+def _get_parakeet_model():
+    global _PARAKEET_MODEL
+    if _PARAKEET_MODEL is not None:
+        return _PARAKEET_MODEL
+
+    try:
+        import nemo.collections.asr as nemo_asr
+    except Exception as exc:
+        raise gr.Error(
+            "NVIDIA Parakeet backend requested but NeMo ASR package is missing. "
+            "Add nemo_toolkit[asr] to requirements.txt"
+        ) from exc
+
+    model = nemo_asr.models.ASRModel.from_pretrained(model_name=MODEL_IDS[PARAKEET_V3])
+    if torch.cuda.is_available():
+        model = model.to("cuda")
+    _PARAKEET_MODEL = model
+    return _PARAKEET_MODEL
+
+
+def run_parakeet(
+    audio_file: str,
+    language: str,
+    model_options: dict,
+) -> dict:
+    model = _get_parakeet_model()
+    batch_size = int(model_options.get("batch_size", 1))
+
+    infer_start = time.perf_counter()
+    outputs = model.transcribe([audio_file], batch_size=batch_size, timestamps=True)
+    infer_end = time.perf_counter()
+
+    item = outputs[0] if outputs else None
+    raw_output = {
+        "output": serialize(item),
+        "timestamp_hint": "word timestamps available in output.timestamp['word'] when provided by NeMo",
+        "language_hint": language or "auto",
+    }
+
+    return {
+        "raw_output": raw_output,
+        "timing": {
+            "inference_seconds": round(infer_end - infer_start, 4),
+        },
+    }

src/models/whisper_cpp_model.py

@@ -0,0 +1,77 @@
+import json
+import os
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+import gradio as gr
+
+from src.utils import serialize
+
+
+def run_whisper_cpp(
+    audio_file: str,
+    task: str,
+    language: str,
+    initial_prompt: str,
+    model_options: dict,
+) -> dict:
+    whisper_cpp_bin = model_options.get("whisper_cpp_bin") or os.getenv("WHISPER_CPP_BIN", "whisper-cli")
+    whisper_cpp_model = model_options.get("whisper_cpp_model") or os.getenv("WHISPER_CPP_MODEL_LARGE")
+    if not whisper_cpp_model:
+        raise gr.Error(
+            "Whisper.cpp requires model path. Set WHISPER_CPP_MODEL_LARGE or pass "
+            "model_options_json={\"whisper_cpp_model\":\"/path/to/ggml-large-v3.bin\"}."
+        )
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_prefix = str(Path(tmpdir) / "whispercpp")
+        cmd = [
+            whisper_cpp_bin,
+            "-m",
+            whisper_cpp_model,
+            "-f",
+            audio_file,
+            "-oj",
+            "-ml",
+            "1",
+            "-of",
+            output_prefix,
+        ]
+
+        if language:
+            cmd.extend(["-l", language])
+        if initial_prompt:
+            cmd.extend(["--prompt", initial_prompt])
+        if task == "translate":
+            cmd.append("-tr")
+
+        infer_start = time.perf_counter()
+        proc = subprocess.run(cmd, capture_output=True, text=True)
+        infer_end = time.perf_counter()
+
+        if proc.returncode != 0:
+            raise gr.Error(
+                "whisper.cpp transcription failed. "
+                f"exit={proc.returncode} stderr={proc.stderr[-1500:]}"
+            )
+
+        json_path = Path(f"{output_prefix}.json")
+        if not json_path.exists():
+            raise gr.Error(
+                "whisper.cpp did not produce JSON output. "
+                "Ensure your whisper.cpp binary supports -oj and word timestamps (-ml 1)."
+            )
+
+        raw_output = json.loads(json_path.read_text())
+
+    return {
+        "raw_output": {
+            "result": serialize(raw_output),
+            "stderr": proc.stderr,
+        },
+        "timing": {
+            "inference_seconds": round(infer_end - infer_start, 4),
+        },
+    }

src/models/whisper_transformers.py

@@ -0,0 +1,76 @@
+import time
+from typing import Any
+
+import torch
+from transformers import pipeline
+
+from src.constants import BATCH_SIZE, MODEL_IDS
+from src.utils import serialize
+
+_TRANSFORMERS_PIPES: dict[str, Any] = {}
+
+
+def _device_for_transformers() -> int | str:
+    return 0 if torch.cuda.is_available() else "cpu"
+
+
+def _get_whisper_pipeline(model_label: str):
+    if model_label in _TRANSFORMERS_PIPES:
+        return _TRANSFORMERS_PIPES[model_label]
+
+    model_name = MODEL_IDS[model_label]
+    dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+    pipe = pipeline(
+        task="automatic-speech-recognition",
+        model=model_name,
+        chunk_length_s=30,
+        batch_size=BATCH_SIZE,
+        device=_device_for_transformers(),
+        model_kwargs={"torch_dtype": dtype, "low_cpu_mem_usage": True},
+    )
+    _TRANSFORMERS_PIPES[model_label] = pipe
+    return pipe
+
+
+def run_whisper_transformers(
+    model_label: str,
+    audio_file: str,
+    task: str,
+    language: str,
+    initial_prompt: str,
+    model_options: dict[str, Any],
+) -> dict[str, Any]:
+    pipe = _get_whisper_pipeline(model_label)
+
+    generate_kwargs: dict[str, Any] = {"task": task}
+    if language:
+        generate_kwargs["language"] = language
+
+    if initial_prompt:
+        try:
+            prompt_ids = pipe.tokenizer.get_prompt_ids(initial_prompt, return_tensors="pt")
+            if hasattr(prompt_ids, "to") and torch.cuda.is_available():
+                prompt_ids = prompt_ids.to("cuda")
+            generate_kwargs["prompt_ids"] = prompt_ids
+        except Exception:
+            generate_kwargs["prompt"] = initial_prompt
+
+    if "temperature" in model_options:
+        generate_kwargs["temperature"] = model_options["temperature"]
+    if "num_beams" in model_options:
+        generate_kwargs["num_beams"] = model_options["num_beams"]
+
+    infer_start = time.perf_counter()
+    raw_output = pipe(
+        audio_file,
+        return_timestamps="word",
+        generate_kwargs=generate_kwargs,
+    )
+    infer_end = time.perf_counter()
+
+    return {
+        "raw_output": serialize(raw_output),
+        "timing": {
+            "inference_seconds": round(infer_end - infer_start, 4),
+        },
+    }

src/transcription_service.py

@@ -0,0 +1,183 @@
+import time
+
+import gradio as gr
+
+from src.constants import (
+    PARAKEET_V3,
+    SUPPORTED_MODELS,
+    WHISPER_CPP_LARGE,
+    WHISPER_FASTER_LARGE,
+    WHISPER_LARGE_V3,
+    WHISPER_LARGE_V3_TURBO,
+)
+from src.models.faster_whisper_model import run_faster_whisper
+from src.models.parakeet_model import run_parakeet
+from src.models.whisper_cpp_model import run_whisper_cpp
+from src.models.whisper_transformers import run_whisper_transformers
+from src.utils import parse_model_options
+
+
+def dispatch_transcription(
+    audio_file: str,
+    model_label: str,
+    task: str,
+    language: str,
+    initial_prompt: str,
+    postprocess_prompt: str,
+    model_options_json: str,
+) -> dict:
+    if audio_file is None:
+        raise gr.Error("No audio file submitted. Upload an audio file first.")
+    if model_label not in SUPPORTED_MODELS:
+        raise gr.Error(f"Model is not supported for word-level timestamps: {model_label}")
+    if task not in {"transcribe", "translate"}:
+        raise gr.Error("task must be one of: transcribe, translate")
+
+    model_options = parse_model_options(model_options_json)
+    return dispatch_transcription_with_options(
+        audio_file=audio_file,
+        model_label=model_label,
+        task=task,
+        language=language,
+        initial_prompt=initial_prompt,
+        postprocess_prompt=postprocess_prompt,
+        model_options=model_options,
+    )
+
+
+def dispatch_transcription_with_options(
+    audio_file: str,
+    model_label: str,
+    task: str,
+    language: str,
+    initial_prompt: str,
+    postprocess_prompt: str,
+    model_options: dict,
+) -> dict:
+    gpu_start = time.perf_counter()
+
+    if model_label in {WHISPER_LARGE_V3, WHISPER_LARGE_V3_TURBO}:
+        result = run_whisper_transformers(
+            model_label=model_label,
+            audio_file=audio_file,
+            task=task,
+            language=language,
+            initial_prompt=initial_prompt,
+            model_options=model_options,
+        )
+    elif model_label == WHISPER_FASTER_LARGE:
+        result = run_faster_whisper(
+            audio_file=audio_file,
+            task=task,
+            language=language,
+            initial_prompt=initial_prompt,
+            model_options=model_options,
+        )
+    elif model_label == WHISPER_CPP_LARGE:
+        result = run_whisper_cpp(
+            audio_file=audio_file,
+            task=task,
+            language=language,
+            initial_prompt=initial_prompt,
+            model_options=model_options,
+        )
+    elif model_label == PARAKEET_V3:
+        if task == "translate":
+            raise gr.Error("NVIDIA Parakeet v3 backend in this app currently supports task='transcribe' only.")
+        result = run_parakeet(
+            audio_file=audio_file,
+            language=language,
+            model_options=model_options,
+        )
+    else:
+        raise gr.Error(f"Unsupported model {model_label}")
+
+    gpu_end = time.perf_counter()
+
+    return {
+        "model": model_label,
+        "task": task,
+        "audio_file": str(audio_file),
+        "postprocess_prompt": postprocess_prompt or None,
+        "model_options": model_options,
+        "zerogpu_timing": {
+            "gpu_window_seconds": round(gpu_end - gpu_start, 4),
+            **result.get("timing", {}),
+        },
+        "raw_output": result["raw_output"],
+        "timestamp_granularity": "word",
+    }
+
+
+def benchmark_all_models(
+    audio_file: str,
+    task: str,
+    language: str,
+    initial_prompt: str,
+    postprocess_prompt: str,
+    model_options_json: str,
+) -> dict:
+    if audio_file is None:
+        raise gr.Error("No audio file submitted. Upload an audio file first.")
+    model_options = parse_model_options(model_options_json)
+
+    started_at = time.perf_counter()
+    results = []
+
+    for model_label in SUPPORTED_MODELS:
+        per_model_start = time.perf_counter()
+        try:
+            model_result = dispatch_transcription_with_options(
+                audio_file=audio_file,
+                model_label=model_label,
+                task=task,
+                language=language,
+                initial_prompt=initial_prompt,
+                postprocess_prompt=postprocess_prompt,
+                model_options=model_options,
+            )
+            per_model_end = time.perf_counter()
+            results.append(
+                {
+                    "model": model_label,
+                    "status": "ok",
+                    "wall_clock_seconds": round(per_model_end - per_model_start, 4),
+                    "result": model_result,
+                }
+            )
+        except Exception as exc:
+            per_model_end = time.perf_counter()
+            results.append(
+                {
+                    "model": model_label,
+                    "status": "error",
+                    "wall_clock_seconds": round(per_model_end - per_model_start, 4),
+                    "error": str(exc),
+                }
+            )
+
+    completed_at = time.perf_counter()
+
+    leaderboard = sorted(
+        [r for r in results if r["status"] == "ok"],
+        key=lambda item: item["result"]["zerogpu_timing"].get("gpu_window_seconds", float("inf")),
+    )
+
+    return {
+        "task": task,
+        "audio_file": str(audio_file),
+        "language": language or None,
+        "timestamp_granularity": "word",
+        "benchmark_timing": {
+            "total_wall_clock_seconds": round(completed_at - started_at, 4),
+        },
+        "results": results,
+        "leaderboard_by_gpu_window_seconds": [
+            {
+                "model": item["model"],
+                "gpu_window_seconds": item["result"]["zerogpu_timing"].get("gpu_window_seconds"),
+                "inference_seconds": item["result"]["zerogpu_timing"].get("inference_seconds"),
+            }
+            for item in leaderboard
+        ],
+    }

src/utils.py

@@ -0,0 +1,41 @@
+import json
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+
+
+def serialize(value: Any) -> Any:
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    if isinstance(value, Path):
+        return str(value)
+    if isinstance(value, dict):
+        return {str(k): serialize(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [serialize(v) for v in value]
+    if hasattr(value, "item"):
+        try:
+            return value.item()
+        except Exception:
+            pass
+    if hasattr(value, "tolist"):
+        try:
+            return value.tolist()
+        except Exception:
+            pass
+    if hasattr(value, "__dict__"):
+        return {k: serialize(v) for k, v in vars(value).items()}
+    return str(value)
+
+
+def parse_model_options(raw: str | None) -> dict[str, Any]:
+    if not raw:
+        return {}
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise gr.Error(f"model_options_json must be valid JSON: {exc}") from exc
+    if not isinstance(parsed, dict):
+        raise gr.Error("model_options_json must decode to a JSON object")
+    return parsed