custom eval update

app.py

@@ -178,6 +178,15 @@ def submit_model(
     if not on_hub:
         return styled_error(f"Model '{model_id}' {err_msg}")
 
+    script_hint = ""
+    if (custom_script or "").strip():
+        if not job_queue.custom_script_defines_evaluate(custom_script):
+            script_hint = (
+                "<p style='color:orange'><strong>Note:</strong> Your custom script should define "
+                "<code>evaluate(file: pathlib.Path) -&gt; str</code> at the top level. "
+                "It will be called once per sample during evaluation.</p>"
+            )
+
     try:
         job_id, position, err, awaiting_mod = job_queue.enqueue(
             model_id,
@@ -226,18 +235,24 @@ def submit_model(
         return styled_warning(err)
 
     if awaiting_mod:
-        return styled_message(
-            f"Request <code>{job_id}</code> recorded for <strong>{model_id}</strong>. "
-            f"<strong>A moderator must approve</strong> it before evaluation starts "
-            f"(see the <strong>Moderate</strong> tab). "
-            f"Approx. backlog awaiting approval: <strong>{position}</strong>."
+        return (
+            script_hint
+            + styled_message(
+                f"Request <code>{job_id}</code> recorded for <strong>{model_id}</strong>. "
+                f"<strong>A moderator must approve</strong> it before evaluation starts "
+                f"(see the <strong>Moderate</strong> tab). "
+                f"Approx. backlog awaiting approval: <strong>{position}</strong>."
+            )
         )
 
-    return styled_message(
-        f"Queued job <code>{job_id}</code> for <strong>{model_id}</strong>. "
-        f"Approx. position in queue: <strong>{position}</strong>. "
-        f"Evaluation runs in the background; refresh the <strong>Leaderboard</strong> tab "
-        f"after a few minutes to see WER when the job finishes."
+    return (
+        script_hint
+        + styled_message(
+            f"Queued job <code>{job_id}</code> for <strong>{model_id}</strong>. "
+            f"Approx. position in queue: <strong>{position}</strong>. "
+            f"Evaluation runs in the background; refresh the <strong>Leaderboard</strong> tab "
+            f"after a few minutes to see WER when the job finishes."
+        )
     )
 
 
@@ -424,13 +439,22 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
                 max_length=8000,
             )
             script_input = gr.Code(
-                label="Optional custom uv-run script (only run if a moderator approves it)",
+                label="Optional custom evaluator (Python)",
                 language="python",
                 lines=12,
             )
             gr.Markdown(
-                "Custom scripts are **not** executed on this Space. They are stored with your submission "
-                "and run on a Hub Job **only** after a moderator explicitly approves using that script."
+                "Define a function that transcribes one WAV file:\n\n"
+                "```python\n"
+                "from pathlib import Path\n\n"
+                "def evaluate(file: Path) -> str:\n"
+                "    # return the transcription for this audio file\n"
+                "    ...\n"
+                "```\n\n"
+                "It is called **once per audio sample** inside the existing eval loop. "
+                "Put any extra Python dependencies in the requirements box above. "
+                "Custom evaluators are **not** run on this Space; they run on a Hub Job "
+                "**only** after a moderator approves them."
             )
             with gr.Row():
                 is_gated_input = gr.Checkbox(label="This is a gated repo", value=False)
@@ -508,7 +532,7 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
                     value=None,
                 )
                 mod_run_custom_script = gr.Checkbox(
-                    label="Run submitter's custom script on approval (if provided)",
+                    label="Use submitter's custom evaluate() function (if provided)",
                     value=False,
                 )
                 with gr.Row():

backends/custom_eval.py

@@ -0,0 +1,96 @@
+"""
+Load a user-submitted custom evaluator script and wrap it as a transcriber callable.
+
+The script must define::
+
+    from pathlib import Path
+    def evaluate(file: Path) -> str:
+        ...
+
+Each packed benchmark sample is written to a temporary WAV file and passed to
+``evaluate`` once per sample inside the existing eval loop.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import shutil
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+
+def build_transcriber_from_custom_script(
+    script_text: str,
+) -> tuple[Callable[..., str], Callable[[], None]]:
+    """
+    Load ``script_text`` as a module and return ``(transcribe, cleanup)``.
+
+    ``transcribe(audio_np, sampling_rate)`` writes a temp WAV and calls
+    ``evaluate(path)`` on the loaded module.
+    """
+    import soundfile as sf
+
+    script_text = (script_text or "").strip()
+    if not script_text:
+        raise RuntimeError("Custom script is empty.")
+
+    fd, script_path = tempfile.mkstemp(prefix="ffasr_custom_eval_", suffix=".py")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            f.write(script_text)
+    except Exception:
+        try:
+            os.unlink(script_path)
+        except FileNotFoundError:
+            pass
+        raise
+
+    spec = importlib.util.spec_from_file_location("ffasr_user_eval", script_path)
+    if spec is None or spec.loader is None:
+        try:
+            os.unlink(script_path)
+        except FileNotFoundError:
+            pass
+        raise RuntimeError("Could not load custom evaluator script.")
+
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+
+    evaluate_fn = getattr(mod, "evaluate", None)
+    if not callable(evaluate_fn):
+        try:
+            os.unlink(script_path)
+        except FileNotFoundError:
+            pass
+        raise RuntimeError(
+            "Custom script must define `evaluate(file: pathlib.Path) -> str`."
+        )
+
+    wav_dir = tempfile.mkdtemp(prefix="ffasr_custom_wav_")
+    counter = {"i": 0}
+
+    def transcribe(audio_np, sampling_rate: int) -> str:
+        counter["i"] += 1
+        wav_path = Path(wav_dir) / f"sample_{counter['i']:08d}.wav"
+        sf.write(str(wav_path), audio_np, int(sampling_rate), subtype="PCM_16")
+        try:
+            text = evaluate_fn(wav_path)
+        finally:
+            try:
+                wav_path.unlink()
+            except FileNotFoundError:
+                pass
+        return str(text or "")
+
+    transcribe._num_params = 0  # type: ignore[attr-defined]
+
+    def cleanup() -> None:
+        shutil.rmtree(wav_dir, ignore_errors=True)
+        try:
+            os.unlink(script_path)
+        except FileNotFoundError:
+            pass
+
+    return transcribe, cleanup

backends/universal.py

@@ -230,6 +230,15 @@ def _normalize_input_features_layout(model, batch: dict) -> dict:
     return batch
 
 
+def _encoder_tensor_from_batch(batch: dict):
+    """Return encoder inputs without using ``or`` on tensors (ambiguous bool)."""
+    if batch.get("input_features") is not None:
+        return batch["input_features"]
+    if batch.get("input_values") is not None:
+        return batch["input_values"]
+    return None
+
+
 def _generate(model, batch: dict, max_new_tokens: int, language: str):
     """Try language-aware generate first (Whisper path); fall back to plain generate."""
     batch = _normalize_input_features_layout(model, batch)
@@ -240,7 +249,7 @@ def _generate(model, batch: dict, max_new_tokens: int, language: str):
         # Cohere's remote generate() path can leave decoder_attention_mask as None,
         # then HF generation tries `decoder_attention_mask.new_ones(...)` and crashes.
         # Passing an explicit one-token decoder mask keeps generation state valid.
-        src = batch.get("input_features") or batch.get("input_values")
+        src = _encoder_tensor_from_batch(batch)
         if src is not None and hasattr(src, "shape"):
             try:
                 batch = dict(batch)

evaluation/orchestrator.py

@@ -57,6 +57,21 @@ def _samples_per_gpu_segment() -> int:
     return max(0, v)
 
 
+def _build_transcriber(
+    family_id: str,
+    model_id: str,
+    device_str: str,
+    device_int: int,
+    custom_script: str = "",
+) -> tuple[Callable[..., str], Callable[[], None]]:
+    """Family backend transcriber, or user ``evaluate(Path)`` script when provided."""
+    if (custom_script or "").strip():
+        from backends.custom_eval import build_transcriber_from_custom_script
+
+        return build_transcriber_from_custom_script(custom_script)
+    return build_transcriber(family_id, model_id, device_str, device_int)
+
+
 def _zerogpu_run_segment(payload: dict[str, Any]) -> dict[str, Any]:
     """Single ZeroGPU segment: build model, transcribe ``[start:end)`` for one condition."""
     model_id = str(payload["model_id"])
@@ -64,9 +79,12 @@ def _zerogpu_run_segment(payload: dict[str, Any]) -> dict[str, Any]:
     condition_key = str(payload["condition_key"])
     start = int(payload["start"])
     end = int(payload["end"])
+    custom_script = str(payload.get("custom_script") or "")
 
     device_str, device_int = resolve_eval_devices()
-    transcribe, cleanup = build_transcriber(family_id, model_id, device_str, device_int)
+    transcribe, cleanup = _build_transcriber(
+        family_id, model_id, device_str, device_int, custom_script=custom_script
+    )
     try:
         num_params = int(getattr(transcribe, "_num_params", 0) or 0)
         preds, refs, audio_s, infer_s, n_done = accumulate_predictions_for_slice(
@@ -90,6 +108,7 @@ def _run_evaluation_core(
     model_id: str,
     family_id: str = "auto",
     progress_cb: Callable[[int, int, str], None] | None = None,
+    custom_script: str = "",
 ) -> dict:
     device_str, device_int = resolve_eval_devices()
 
@@ -109,7 +128,9 @@ def _run_evaluation_core(
         except Exception:
             pass
 
-    transcribe, cleanup = build_transcriber(family_id, model_id, device_str, device_int)
+    transcribe, cleanup = _build_transcriber(
+        family_id, model_id, device_str, device_int, custom_script=custom_script
+    )
     try:
         # Backends attach `transcribe._num_params` via `_model_utils.attach_params`; defaults to 0
         # if the backend couldn't introspect a module.
@@ -171,6 +192,7 @@ def _run_evaluation_core_segmented_local(
     progress_cb: Callable[[int, int, str], None] | None = None,
     *,
     segment_samples: int,
+    custom_script: str = "",
 ) -> dict:
     """
     Same semantics as ``_run_evaluation_core_segmented`` but without ``spaces.GPU``:
@@ -224,6 +246,7 @@ def _run_evaluation_core_segmented_local(
                 "condition_key": condition_key,
                 "start": start,
                 "end": end,
+                "custom_script": custom_script,
             }
             out = _zerogpu_run_segment(payload)
             if first_num_params is None:
@@ -270,6 +293,7 @@ def _run_evaluation_core_segmented(
     segment_samples: int,
     gpu_duration_s: int,
     gpu_size: str,
+    custom_script: str = "",
 ) -> dict:
     """
     Long evaluations: each slice of ``segment_samples`` runs under its own ``spaces.GPU``
@@ -327,6 +351,7 @@ def _run_evaluation_core_segmented(
                 "condition_key": condition_key,
                 "start": start,
                 "end": end,
+                "custom_script": custom_script,
             }
             out = wrapped_seg(payload)
             if first_num_params is None:
@@ -383,6 +408,7 @@ def run_evaluation(
     model_id: str,
     family_id: str = "auto",
     progress_cb: Callable[[int, int, str], None] | None = None,
+    custom_script: str = "",
 ) -> dict:
     """
     Evaluate `model_id` on all packed conditions using the selected family backend.
@@ -410,6 +436,9 @@ def run_evaluation(
     ``progress_cb(samples_done_across_all_conditions, samples_total_across_all_conditions, current_condition_key)``
     periodically during the run.
 
+    When ``custom_script`` is non-empty, it must define ``evaluate(file: pathlib.Path) -> str``;
+    that function is called once per packed sample (via a temp WAV) instead of the family backend.
+
     Returns wer_clean, wer_noisy, wer_reverberant, wer_real, wer_difficult, num_samples, model_id, eval_family, plus timing.
     """
     apply_cpu_thread_settings_once()
@@ -423,8 +452,14 @@ def run_evaluation(
                 family_id=family_id,
                 progress_cb=progress_cb,
                 segment_samples=seg,
+                custom_script=custom_script,
             )
-        return _run_evaluation_core(model_id, family_id=family_id, progress_cb=progress_cb)
+        return _run_evaluation_core(
+            model_id,
+            family_id=family_id,
+            progress_cb=progress_cb,
+            custom_script=custom_script,
+        )
 
     eff = _effective_gpu_duration_s()
     size = os.environ.get("FFASR_ZEROGPU_GPU_SIZE", "large").strip().lower()
@@ -433,7 +468,12 @@ def run_evaluation(
 
     if seg <= 0:
         wrapped = _spaces_gpu_wrapped(eff, size)
-        return wrapped(model_id, family_id=family_id, progress_cb=progress_cb)
+        return wrapped(
+            model_id,
+            family_id=family_id,
+            progress_cb=progress_cb,
+            custom_script=custom_script,
+        )
 
     return _run_evaluation_core_segmented(
         model_id,
@@ -442,4 +482,5 @@ def run_evaluation(
         segment_samples=seg,
         gpu_duration_s=eff,
         gpu_size=size,
+        custom_script=custom_script,
     )

job_queue.py

@@ -138,6 +138,23 @@ def sanitize_custom_script(text: str) -> str:
     return s
 
 
+def custom_script_defines_evaluate(text: str) -> bool:
+    """Best-effort check that the script defines a top-level ``evaluate`` function."""
+    s = (text or "").strip()
+    if not s:
+        return True
+    try:
+        import ast
+
+        tree = ast.parse(s)
+    except SyntaxError:
+        return False
+    return any(
+        isinstance(node, ast.FunctionDef) and node.name == "evaluate"
+        for node in tree.body
+    )
+
+
 def _bool_to_csv(flag: bool) -> str:
     return "1" if flag else "0"
 
@@ -1262,7 +1279,7 @@ def next_up_html(limit: int = 5) -> str:
     return (
         "<div class='next-up-panel' style='font-size:0.9em'>"
         f"<p><strong>Next models to evaluate</strong> ({len(queued)} shown):</p>"
-        f"<ol style='margin:0.25rem 0 0 1.1rem'>{items}</ol></motionmotionmotionmotiondiv>"
+        f"<ol style='margin:0.25rem 0 0 1.1rem'>{items}</ol></div>"
     )
 
 

remote_jobs.py

@@ -187,21 +187,7 @@ def submit_eval_job(
         )
 
     script = _worker_script_path()
-    custom_script_path: str | None = None
-    if run_custom_script and (custom_script or "").strip():
-        import tempfile
-
-        fd, custom_script_path = tempfile.mkstemp(
-            prefix=f"ffasr_custom_{space_job_id}_", suffix=".py"
-        )
-        try:
-            with os.fdopen(fd, "w", encoding="utf-8") as f:
-                f.write(custom_script.strip())
-        except Exception:
-            os.unlink(custom_script_path)
-            raise
-        script = custom_script_path
-    elif not os.path.isfile(script):
+    if not os.path.isfile(script):
         raise RuntimeError(f"Remote worker script not found: {script}")
 
     deps = _select_deps(model_id, family_id)
@@ -233,6 +219,12 @@ def submit_eval_job(
         "FFASR_REMOTE_EVAL_GIT_BRANCH": eval_branch,
         "HF_HUB_DISABLE_PROGRESS_BARS": "1",
     }
+    if run_custom_script and (custom_script or "").strip():
+        import base64
+
+        env["FFASR_CUSTOM_SCRIPT_B64"] = base64.b64encode(
+            custom_script.strip().encode("utf-8")
+        ).decode("ascii")
     secrets = {"HF_TOKEN": bucket_tok}
 
     api = HfApi(token=jobs_tok)
@@ -257,12 +249,6 @@ def submit_eval_job(
         if last_err is not None:
             raise last_err
         raise RuntimeError("submit_eval_job: unreachable")
-    finally:
-        if custom_script_path and os.path.isfile(custom_script_path):
-            try:
-                os.unlink(custom_script_path)
-            except Exception:
-                pass
 
 
 def inspect_job_once(hf_job_id: str, *, token: str | None = None) -> JobInfo:

scripts/run_hf_remote_job_uv.py

@@ -16,6 +16,7 @@ Environment (set by ``remote_jobs.submit_eval_job``):
 
 from __future__ import annotations
 
+import base64
 import io
 import json
 import os
@@ -113,7 +114,16 @@ def main() -> int:
         from storage import HF_BUCKET_ID, upload_to_bucket
 
         apply_cpu_thread_settings_once()
-        result = run_evaluation(model_id, family_id=family_id, progress_cb=None)
+        custom_b64 = os.environ.get("FFASR_CUSTOM_SCRIPT_B64", "").strip()
+        custom_script = (
+            base64.b64decode(custom_b64).decode("utf-8") if custom_b64 else ""
+        )
+        result = run_evaluation(
+            model_id,
+            family_id=family_id,
+            progress_cb=None,
+            custom_script=custom_script,
+        )
         artifact = build_artifact(
             model_id=model_id,
             family_id=family_id,