cohere fix

app.py

@@ -460,6 +460,8 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
             gr.Markdown(
                 "Edit the example below: define **`evaluate(file: Path) -> str`** to transcribe one WAV. "
                 "Load your model **once** at module level (as in the Cohere example). "
+                "Use **soundfile** (or numpy) to read each WAV — avoid "
+                "``transformers.audio_utils.load_audio`` (requires FFmpeg/torchcodec on the job). "
                 "The function is called **once per audio sample** inside the eval loop. "
                 "Put extra Python dependencies in the requirements box above. "
                 "Custom evaluators run on a Hub Job **only** after a moderator approves them."

backends/_audio_utils.py

@@ -4,6 +4,8 @@ Small helpers shared across backends (no heavy imports).
 
 from __future__ import annotations
 
+from pathlib import Path
+
 import numpy as np
 
 
@@ -26,3 +28,29 @@ def safe_pad_audio(audio: np.ndarray, multiple: int = 1600) -> np.ndarray:
         return arr
     pad = multiple - rem
     return np.concatenate([arr, np.zeros(pad, dtype=np.float32)])
+
+
+def load_wav_mono(path: str | Path, sampling_rate: int = 16000) -> np.ndarray:
+    """
+    Load a WAV file as a 1-D float32 mono waveform at ``sampling_rate`` Hz.
+
+    Uses ``soundfile`` only (no torchcodec / FFmpeg). Eval samples and the
+    custom ``evaluate(Path)`` hook are written as 16 kHz PCM WAVs.
+    """
+    import soundfile as sf
+
+    audio, sr = sf.read(str(path), dtype="float32", always_2d=True)
+    audio = audio.mean(axis=1)
+    if int(sr) != int(sampling_rate):
+        try:
+            import librosa
+
+            audio = librosa.resample(
+                audio, orig_sr=int(sr), target_sr=int(sampling_rate)
+            )
+        except Exception as exc:
+            raise RuntimeError(
+                f"Audio is {sr} Hz but {sampling_rate} Hz was requested; "
+                "install librosa for resampling."
+            ) from exc
+    return np.asarray(audio, dtype=np.float32).reshape(-1)

constants.py

@@ -418,8 +418,10 @@ button.primary, .primary button {
 )
 
 DEFAULT_CUSTOM_EVAL_EXAMPLE = """from pathlib import Path
+
+import soundfile as sf
+import torch
 from transformers import AutoProcessor, CohereAsrForConditionalGeneration
-from transformers.audio_utils import load_audio
 
 processor = AutoProcessor.from_pretrained("CohereLabs/cohere-transcribe-03-2026")
 model = CohereAsrForConditionalGeneration.from_pretrained(
@@ -427,8 +429,9 @@ model = CohereAsrForConditionalGeneration.from_pretrained(
 )
 
 def evaluate(file: Path) -> str:
-    audio = load_audio(file, sampling_rate=16000)
-    inputs = processor(audio, sampling_rate=16000, return_tensors="pt", language="en")
+    audio, sr = sf.read(str(file), dtype="float32", always_2d=True)
+    audio = audio.mean(axis=1)
+    inputs = processor(audio, sampling_rate=int(sr), return_tensors="pt", language="en")
     inputs.to(model.device, dtype=model.dtype)
     outputs = model.generate(**inputs, max_new_tokens=256)
     return processor.decode(outputs, skip_special_tokens=True)

evaluation/orchestrator.py

@@ -442,6 +442,7 @@ def run_evaluation(
     Returns wer_clean, wer_noisy, wer_reverberant, wer_real, wer_difficult, num_samples, model_id, eval_family, plus timing.
     """
     apply_cpu_thread_settings_once()
+    disable_broken_torchcodec()
 
     seg = _samples_per_gpu_segment()
 

evaluation/runtime.py

@@ -6,6 +6,35 @@ import importlib.util
 import os
 
 _cpu_thread_settings_applied = False
+_torchcodec_patch_applied = False
+
+
+def disable_broken_torchcodec() -> None:
+    """
+    Transformers prefers torchcodec for ``load_audio`` when the package is installed,
+    but Hub Job images often lack FFmpeg shared libraries (libavutil.so.*).
+
+    If torchcodec cannot load, mark it unavailable so transformers falls back to
+    librosa / soundfile.
+    """
+    global _torchcodec_patch_applied
+    if _torchcodec_patch_applied:
+        return
+    _torchcodec_patch_applied = True
+
+    import importlib.util
+
+    if importlib.util.find_spec("torchcodec") is None:
+        return
+    try:
+        from torchcodec.decoders import AudioDecoder  # noqa: F401
+    except Exception:
+        try:
+            import transformers.utils.import_utils as import_utils
+
+            import_utils._torchcodec_available = False
+        except Exception:
+            pass
 
 
 def zerogpu_disabled() -> bool:

remote_jobs.py

@@ -29,6 +29,7 @@ _DEFAULT_REMOTE_JOB_FLAVOR = "l4x1"
 _DEFAULT_REMOTE_WORKER_DEVICE = "auto"
 
 # Shared eval stack installed by uv for every remote job.
+# Note: do not add torchcodec — Hub Job images lack FFmpeg shared libs (libavutil.so.*).
 CORE_DEPS: list[str] = [
     "pandas",
     "numpy",
@@ -43,7 +44,6 @@ CORE_DEPS: list[str] = [
 
 _TRANSFORMERS_DEPS: list[str] = [
     "transformers @ git+https://github.com/huggingface/transformers.git",
-    "torchcodec",
     "librosa",
     "sentencepiece",
 ]

scripts/run_eval_and_publish.py

@@ -460,6 +460,10 @@ def _run_evaluation_local(
     """Inline orchestrator: chosen device + batched fast-path when possible."""
     import torch
 
+    from evaluation.runtime import disable_broken_torchcodec
+
+    disable_broken_torchcodec()
+
     from backends.registry import build_transcriber, resolve_label
     from benchmark.dataset import (
         PACKED_FILES,

scripts/run_hf_remote_job_uv.py

@@ -109,11 +109,12 @@ def main() -> int:
         os.chdir(workdir)
 
         from evaluation.remote_artifact import ARTIFACT_SCHEMA_VERSION, build_artifact
-        from evaluation.runtime import apply_cpu_thread_settings_once
+        from evaluation.runtime import apply_cpu_thread_settings_once, disable_broken_torchcodec
         from evaluation.orchestrator import run_evaluation
         from storage import HF_BUCKET_ID, upload_to_bucket
 
         apply_cpu_thread_settings_once()
+        disable_broken_torchcodec()
         custom_b64 = os.environ.get("FFASR_CUSTOM_SCRIPT_B64", "").strip()
         custom_script = (
             base64.b64decode(custom_b64).decode("utf-8") if custom_b64 else ""