selected dataset eval

app.py

@@ -521,7 +521,16 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
                     "When <code>FFASR_REMOTE_JOBS=1</code>, use **Open Hub Job logs** while a job runs. "
                     "The bucket <code>results/remote_artifacts/</code> folder only gets a JSON file **after** a successful run.\n\n"
                     "**Retry** re-queues failed, done, or queued jobs (not while running). "
-                    "A successful re-run **replaces** the leaderboard row for that model if it already exists."
+                    "Use **Datasets to evaluate** to run only selected packed splits; "
+                    "other leaderboard WER columns are left unchanged on success. "
+                    "A full re-run (all datasets selected) **replaces** the row when the model is already listed."
+                )
+                from benchmark.dataset import CONDITION_UI_CHOICES
+
+                mod_eval_conditions = gr.CheckboxGroup(
+                    choices=[(lbl, key) for lbl, key in CONDITION_UI_CHOICES],
+                    value=[key for _, key in CONDITION_UI_CHOICES],
+                    label="Datasets to evaluate (uncheck to skip; all selected = full benchmark)",
                 )
                 gr.Markdown("### Current job progress")
                 mod_progress = gr.HTML(
@@ -667,30 +676,35 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
                     *data,
                 )
 
-            def _mod_approve(secret, jid, unlocked, run_custom):
+            def _mod_approve(secret, jid, unlocked, run_custom, eval_conds):
                 if not unlocked:
                     return _mod_bundle("<p style='color:red'>Unlock moderator tools first.</p>")
                 if not jid or not str(jid).strip():
                     return _mod_bundle("<p style='color:red'>Select a job.</p>")
-
-                def _approve(j, s):
-                    return job_queue.approve_job(j, s, run_custom_script=bool(run_custom))
-
-                ok, msg = _approve(str(jid).strip(), secret)
+                ok, msg = job_queue.approve_job(
+                    str(jid).strip(),
+                    secret,
+                    run_custom_script=bool(run_custom),
+                    eval_conditions=list(eval_conds or []),
+                )
                 color = "green" if ok else "red"
                 return _mod_bundle(f"<p style='color:{color}'>{msg}</p>")
 
             def _mod_reject(secret, jid, unlocked):
                 return _mod_run(job_queue.reject_job, secret, jid, "Select a job.", unlocked)
 
-            def _mod_retry(secret, jid, unlocked):
-                return _mod_run(
-                    job_queue.retry_failed_job,
+            def _mod_retry(secret, jid, unlocked, eval_conds):
+                if not unlocked:
+                    return _mod_bundle("<p style='color:red'>Unlock moderator tools first.</p>")
+                if not jid or not str(jid).strip():
+                    return _mod_bundle("<p style='color:red'>Select a job in Retry / remove.</p>")
+                ok, msg = job_queue.retry_failed_job(
+                    str(jid).strip(),
                     secret,
-                    jid,
-                    "Select a job in Retry / remove.",
-                    unlocked,
+                    eval_conditions=list(eval_conds or []),
                 )
+                color = "green" if ok else "red"
+                return _mod_bundle(f"<p style='color:{color}'>{msg}</p>")
 
             def _mod_remove(secret, jid, unlocked):
                 return _mod_run(
@@ -745,7 +759,13 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
             )
             approve_btn.click(
                 fn=_mod_approve,
-                inputs=[mod_secret, job_pick, mod_unlocked, mod_run_custom_script],
+                inputs=[
+                    mod_secret,
+                    job_pick,
+                    mod_unlocked,
+                    mod_run_custom_script,
+                    mod_eval_conditions,
+                ],
                 outputs=_MOD_OUTS,
             )
             reject_btn.click(
@@ -755,7 +775,7 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
             )
             retry_job_btn.click(
                 fn=_mod_retry,
-                inputs=[mod_secret, mod_action_job, mod_unlocked],
+                inputs=[mod_secret, mod_action_job, mod_unlocked, mod_eval_conditions],
                 outputs=_MOD_OUTS,
             )
             remove_job_btn.click(

benchmark/__init__.py

@@ -1,17 +1,25 @@
 """Shared evaluation data loading and metrics (WER over packed tensors)."""
 
 from .dataset import (
-    PACKED_FILES,
+    CONDITION_UI_CHOICES,
     CONDITION_WER_SUFFIX,
+    DEFAULT_CONDITION_KEYS,
+    PACKED_FILES,
     evaluate_condition_wer,
+    is_full_condition_set,
     load_packed_condition,
+    resolve_condition_keys,
     wer_result_key,
 )
 
 __all__ = [
-    "PACKED_FILES",
+    "CONDITION_UI_CHOICES",
     "CONDITION_WER_SUFFIX",
+    "DEFAULT_CONDITION_KEYS",
+    "PACKED_FILES",
     "evaluate_condition_wer",
+    "is_full_condition_set",
     "load_packed_condition",
+    "resolve_condition_keys",
     "wer_result_key",
 ]

benchmark/dataset.py

@@ -37,6 +37,31 @@ def wer_result_key(condition_key: str) -> str:
     return f"wer_{suffix}"
 
 
+# Gradio checkbox labels (display, condition_key).
+CONDITION_UI_CHOICES: tuple[tuple[str, str], ...] = (
+    ("Dry Speech", "clean"),
+    ("Lab Measured", "measured"),
+    ("Lab Simulated", "sim"),
+    ("High SNR", "high"),
+    ("Mid SNR", "mid"),
+    ("Low SNR", "low"),
+)
+
+DEFAULT_CONDITION_KEYS: tuple[str, ...] = tuple(PACKED_FILES.keys())
+
+
+def resolve_condition_keys(keys: list[str] | None) -> tuple[str, ...]:
+    """Return validated condition keys; ``None`` or empty means all packed splits."""
+    if not keys:
+        return DEFAULT_CONDITION_KEYS
+    valid = [k for k in keys if k in PACKED_FILES]
+    return tuple(valid) if valid else DEFAULT_CONDITION_KEYS
+
+
+def is_full_condition_set(keys: list[str] | None) -> bool:
+    return set(resolve_condition_keys(keys)) == set(PACKED_FILES.keys())
+
+
 _dataset_cache: dict[str, dict] = {}
 
 

evaluation/orchestrator.py

@@ -10,10 +10,11 @@ from typing import Any
 import torch
 
 from benchmark.dataset import (
-    PACKED_FILES,
     accumulate_predictions_for_slice,
     evaluate_condition_wer_timed,
+    is_full_condition_set,
     load_packed_condition,
+    resolve_condition_keys,
     wer_result_key,
 )
 from backends.registry import build_transcriber, resolve_label
@@ -110,13 +111,15 @@ def _run_evaluation_core(
     family_id: str = "auto",
     progress_cb: Callable[[int, int, str], None] | None = None,
     custom_script: str = "",
+    condition_keys: list[str] | None = None,
 ) -> dict:
     device_str, device_int = resolve_eval_devices()
+    keys = resolve_condition_keys(condition_keys)
 
     # Precompute totals so the progress bar has a known denominator from the start.
     condition_totals: dict[str, int] = {}
     grand_total = 0
-    for ck in PACKED_FILES:
+    for ck in keys:
         try:
             condition_totals[ck] = len(load_packed_condition(ck))
         except Exception:
@@ -146,7 +149,7 @@ def _run_evaluation_core(
         total_infer_s = 0.0
         done_so_far = 0
 
-        for condition_key in PACKED_FILES:
+        for condition_key in keys:
             condition_base = done_so_far
 
             def _sample_cb(ck: str, i: int, _n: int, _base: int = condition_base) -> None:
@@ -166,6 +169,8 @@ def _run_evaluation_core(
             total_infer_s += infer_s
             done_so_far += count
 
+        results["eval_conditions"] = list(keys)
+        results["eval_full_run"] = is_full_condition_set(condition_keys)
         results["num_samples"] = total_samples
         results["eval_audio_seconds"] = round(total_audio_s, 3)
         results["eval_wall_time_s"] = round(total_infer_s, 3)
@@ -194,6 +199,7 @@ def _run_evaluation_core_segmented_local(
     *,
     segment_samples: int,
     custom_script: str = "",
+    condition_keys: list[str] | None = None,
 ) -> dict:
     """
     Same semantics as ``_run_evaluation_core_segmented`` but without ``spaces.GPU``:
@@ -201,9 +207,10 @@ def _run_evaluation_core_segmented_local(
     """
     from jiwer import wer as compute_wer
 
+    keys = resolve_condition_keys(condition_keys)
     condition_totals: dict[str, int] = {}
     grand_total = 0
-    for ck in PACKED_FILES:
+    for ck in keys:
         try:
             condition_totals[ck] = len(load_packed_condition(ck))
         except Exception:
@@ -227,7 +234,7 @@ def _run_evaluation_core_segmented_local(
     done_so_far = 0
     first_num_params: int | None = None
 
-    for condition_key in PACKED_FILES:
+    for condition_key in keys:
         data = load_packed_condition(condition_key)
         n_items = len(data)
         all_p: list[str] = []
@@ -270,6 +277,8 @@ def _run_evaluation_core_segmented_local(
         total_samples = max(total_samples, len(all_p))
 
     results["num_params"] = first_num_params or 0
+    results["eval_conditions"] = list(keys)
+    results["eval_full_run"] = is_full_condition_set(condition_keys)
     results["num_samples"] = total_samples
     results["eval_audio_seconds"] = round(total_audio_s, 3)
     results["eval_wall_time_s"] = round(total_infer_s, 3)
@@ -295,6 +304,7 @@ def _run_evaluation_core_segmented(
     gpu_duration_s: int,
     gpu_size: str,
     custom_script: str = "",
+    condition_keys: list[str] | None = None,
 ) -> dict:
     """
     Long evaluations: each slice of ``segment_samples`` runs under its own ``spaces.GPU``
@@ -304,9 +314,10 @@ def _run_evaluation_core_segmented(
 
     import spaces
 
+    keys = resolve_condition_keys(condition_keys)
     condition_totals: dict[str, int] = {}
     grand_total = 0
-    for ck in PACKED_FILES:
+    for ck in keys:
         try:
             condition_totals[ck] = len(load_packed_condition(ck))
         except Exception:
@@ -332,7 +343,7 @@ def _run_evaluation_core_segmented(
     done_so_far = 0
     first_num_params: int | None = None
 
-    for condition_key in PACKED_FILES:
+    for condition_key in keys:
         data = load_packed_condition(condition_key)
         n_items = len(data)
         all_p: list[str] = []
@@ -375,6 +386,8 @@ def _run_evaluation_core_segmented(
         total_samples = max(total_samples, len(all_p))
 
     results["num_params"] = first_num_params or 0
+    results["eval_conditions"] = list(keys)
+    results["eval_full_run"] = is_full_condition_set(condition_keys)
     results["num_samples"] = total_samples
     results["eval_audio_seconds"] = round(total_audio_s, 3)
     results["eval_wall_time_s"] = round(total_infer_s, 3)
@@ -410,6 +423,7 @@ def run_evaluation(
     family_id: str = "auto",
     progress_cb: Callable[[int, int, str], None] | None = None,
     custom_script: str = "",
+    condition_keys: list[str] | None = None,
 ) -> dict:
     """
     Evaluate `model_id` on all packed conditions using the selected family backend.
@@ -455,12 +469,14 @@ def run_evaluation(
                 progress_cb=progress_cb,
                 segment_samples=seg,
                 custom_script=custom_script,
+                condition_keys=condition_keys,
             )
         return _run_evaluation_core(
             model_id,
             family_id=family_id,
             progress_cb=progress_cb,
             custom_script=custom_script,
+            condition_keys=condition_keys,
         )
 
     eff = _effective_gpu_duration_s()
@@ -475,6 +491,7 @@ def run_evaluation(
             family_id=family_id,
             progress_cb=progress_cb,
             custom_script=custom_script,
+            condition_keys=condition_keys,
         )
 
     return _run_evaluation_core_segmented(
@@ -485,4 +502,5 @@ def run_evaluation(
         gpu_duration_s=eff,
         gpu_size=size,
         custom_script=custom_script,
+        condition_keys=condition_keys,
     )

init.py

@@ -139,6 +139,63 @@ def leaderboard_row_from_eval_result(
     return row
 
 
+def merge_leaderboard_row_from_eval_result(
+    result: dict,
+    submitted_at_iso: str,
+    submission_notes: str = "",
+) -> None:
+    """
+    Merge evaluation output into an existing leaderboard row (partial re-eval).
+
+    Only scenario WER columns and ``eval_family`` / ``num_params`` present in
+    ``result`` are overwritten; other metrics and timing fields are kept.
+    Creates a new row if the model is not on the leaderboard yet.
+    """
+    from metrics_config import SCENARIO_KEYS
+
+    model_id = str(result.get("model_id", "")).strip()
+    if not model_id:
+        raise ValueError("merge_leaderboard_row_from_eval_result requires model_id")
+
+    rows = load_raw_results()
+    match_idx: int | None = None
+    for i, r in enumerate(rows):
+        if (r.get("model_id") or "").strip() == model_id:
+            match_idx = i
+            break
+
+    patch = leaderboard_row_from_eval_result(
+        result, submitted_at_iso, submission_notes=submission_notes
+    )
+
+    if match_idx is None:
+        normalize_legacy_csv_row(patch)
+        rows.append(patch)
+    else:
+        row = dict(rows[match_idx])
+        for k in SCENARIO_KEYS:
+            v = patch.get(k)
+            if v is not None and str(v).strip() != "":
+                row[k] = v
+        if patch.get("eval_family"):
+            row["eval_family"] = patch["eval_family"]
+        if patch.get("num_params"):
+            row["num_params"] = patch["num_params"]
+        if (submission_notes or "").strip():
+            row["submission_notes"] = patch["submission_notes"]
+        normalize_legacy_csv_row(row)
+        rows[match_idx] = row
+
+    from analytics import compute_far_field_score_map
+
+    scores = compute_far_field_score_map(rows)
+    rows.sort(
+        key=lambda r: (-scores.get((r.get("model_id") or "").strip(), 0.0), r.get("model_id") or "")
+    )
+    save_raw_results(rows)
+    invalidate_results_cache()
+
+
 # ---------------------------------------------------------------------------
 # Storage helpers
 # ---------------------------------------------------------------------------

job_queue.py

@@ -74,6 +74,8 @@ class Job:
     remote_artifact_path: str | None = None
     # Set by moderator retry: skip duplicate guard and replace CSV row on success.
     replace_leaderboard: bool = False
+    # Subset of PACKED_FILES keys; None means evaluate all conditions.
+    eval_conditions: tuple[str, ...] | None = None
 
 
 _jobs: dict[str, Job] = {}
@@ -110,6 +112,7 @@ _JOBS_CSV_FIELDS = [
     "run_custom_script",
     "hf_remote_job_id",
     "remote_artifact_path",
+    "eval_conditions",
 ]
 
 
@@ -258,6 +261,9 @@ def _persist_jobs() -> None:
                     "run_custom_script": _bool_to_csv(j.run_custom_script),
                     "hf_remote_job_id": (j.hf_remote_job_id or "").strip(),
                     "remote_artifact_path": (j.remote_artifact_path or "").strip(),
+                    "eval_conditions": ",".join(j.eval_conditions)
+                    if j.eval_conditions
+                    else "",
                 }
             )
         rows.sort(key=lambda r: r["created_at"])
@@ -329,6 +335,7 @@ def _load_persisted_jobs_once() -> None:
             fid = (row.get("family_id") or "").strip()
             hf_rid = (row.get("hf_remote_job_id") or "").strip() or None
             art_path = (row.get("remote_artifact_path") or "").strip() or None
+            eval_conds = _parse_eval_conditions_csv(row.get("eval_conditions") or "")
             if not mid:
                 continue
 
@@ -357,6 +364,7 @@ def _load_persisted_jobs_once() -> None:
                 run_custom_script=run_custom,
                 hf_remote_job_id=hf_rid,
                 remote_artifact_path=art_path,
+                eval_conditions=eval_conds,
             )
             _jobs[jid] = job
 
@@ -488,15 +496,23 @@ def _merge_eval_result_to_leaderboard(
     submission_notes: str,
     *,
     replace_existing: bool = False,
+    merge_partial: bool = False,
 ) -> None:
     from init import (
         invalidate_results_cache,
         leaderboard_row_from_eval_result,
         load_raw_results,
+        merge_leaderboard_row_from_eval_result,
         normalize_legacy_csv_row,
         save_raw_results,
     )
 
+    if merge_partial:
+        merge_leaderboard_row_from_eval_result(
+            result, submitted_at_iso, submission_notes=submission_notes
+        )
+        return
+
     model_id = str(result.get("model_id", "")).strip()
     rows = load_raw_results()
     if replace_existing and model_id:
@@ -514,6 +530,55 @@ def _merge_eval_result_to_leaderboard(
     invalidate_results_cache()
 
 
+def _parse_eval_conditions_csv(raw: str) -> tuple[str, ...] | None:
+    from benchmark.dataset import resolve_condition_keys
+
+    parts = [p.strip() for p in (raw or "").split(",") if p.strip()]
+    if not parts:
+        return None
+    keys = resolve_condition_keys(parts)
+    from benchmark.dataset import PACKED_FILES
+
+    if set(keys) == set(PACKED_FILES.keys()):
+        return None
+    return keys
+
+
+def _encode_eval_conditions_csv(keys: tuple[str, ...] | None) -> str:
+    if not keys:
+        return ""
+    return ",".join(keys)
+
+
+def normalize_moderator_eval_conditions(
+    selected: list[str] | None,
+) -> tuple[str, ...] | None:
+    """``None`` = all packed conditions; otherwise a validated non-empty subset."""
+    from benchmark.dataset import PACKED_FILES, resolve_condition_keys
+
+    if not selected:
+        return None
+    keys = resolve_condition_keys(selected)
+    if not keys:
+        raise ValueError("Select at least one dataset to evaluate.")
+    if set(keys) == set(PACKED_FILES.keys()):
+        return None
+    return keys
+
+
+def eval_condition_checkbox_defaults() -> tuple[list[str], list[str]]:
+    """(labels, values) for Gradio CheckboxGroup defaults (all selected)."""
+    from benchmark.dataset import CONDITION_UI_CHOICES
+
+    labels = [lbl for lbl, _ in CONDITION_UI_CHOICES]
+    values = [key for _, key in CONDITION_UI_CHOICES]
+    return labels, values
+
+
+def _job_is_partial_eval(job: Job | None) -> bool:
+    return bool(job and job.eval_conditions)
+
+
 def _load_artifact_json_from_bucket(artifact_path: str) -> dict:
     """Download and parse a remote eval JSON artifact from the Hub bucket."""
     from storage import STORAGE_BACKEND, download_bucket_file
@@ -727,6 +792,7 @@ def _remote_dispatch_job(job_id: str, mid: str, fid: str, jobs_token: str) -> st
         extra_req = (j_dispatch.extra_requirements or "") if j_dispatch else ""
         custom_script = (j_dispatch.custom_script or "") if j_dispatch else ""
         run_custom = bool(j_dispatch and j_dispatch.run_custom_script)
+        eval_conds = j_dispatch.eval_conditions if j_dispatch else None
 
     info = remote_jobs.submit_eval_job(
         model_id=mid,
@@ -737,6 +803,7 @@ def _remote_dispatch_job(job_id: str, mid: str, fid: str, jobs_token: str) -> st
         extra_requirements=extra_req,
         custom_script=custom_script,
         run_custom_script=run_custom,
+        eval_conditions=eval_conds,
     )
     hf_id = info.id
     with _jobs_lock:
@@ -776,7 +843,8 @@ def _remote_start_queued_job(job_id: str, jobs_token: str) -> None:
     with _jobs_lock:
         j_chk = _jobs.get(job_id)
         replace_lb = bool(j_chk and j_chk.replace_leaderboard)
-    if not replace_lb and any(r["model_id"] == mid for r in rows):
+        partial = _job_is_partial_eval(j_chk)
+    if not replace_lb and not partial and any(r["model_id"] == mid for r in rows):
         _fail_job(job_id, "Model already on leaderboard (skipped duplicate race).")
         return
 
@@ -836,8 +904,13 @@ def _remote_tick_job(job_id: str, jobs_token: str) -> None:
         with _jobs_lock:
             j_done = _jobs.get(job_id)
             replace_lb = bool(j_done and j_done.replace_leaderboard)
+            partial = _job_is_partial_eval(j_done)
         _merge_eval_result_to_leaderboard(
-            result, _now_iso(), notes, replace_existing=replace_lb
+            result,
+            _now_iso(),
+            notes,
+            replace_existing=replace_lb and not partial,
+            merge_partial=partial,
         )
         with _jobs_lock:
             j_clr = _jobs.get(job_id)
@@ -1026,7 +1099,11 @@ def enqueue(
 
 
 def approve_job(
-    job_id: str, secret: str, *, run_custom_script: bool = False
+    job_id: str,
+    secret: str,
+    *,
+    run_custom_script: bool = False,
+    eval_conditions: list[str] | None = None,
 ) -> tuple[bool, str]:
     """Move a pending job into the execution queue (moderator only)."""
     if not moderation_active():
@@ -1037,6 +1114,11 @@ def approve_job(
     if not job_id:
         return False, "Enter a job ID."
 
+    try:
+        conds = normalize_moderator_eval_conditions(eval_conditions)
+    except ValueError as e:
+        return False, str(e)
+
     with _jobs_lock:
         job = _jobs.get(job_id)
         if not job or job.status != JobStatus.pending_moderation:
@@ -1049,6 +1131,8 @@ def approve_job(
             job.run_custom_script = bool(run_custom_script)
         else:
             job.run_custom_script = False
+        job.eval_conditions = conds
+        job.replace_leaderboard = False
         _touch(job)
 
     _work_queue.put(job_id)
@@ -1056,13 +1140,16 @@ def approve_job(
     _ensure_worker()
     max_n = remote_max_concurrent_jobs()
     custom_note = ""
+    cond_note = ""
     with _jobs_lock:
         j = _jobs.get(job_id)
         if j and j.run_custom_script and (j.custom_script or "").strip():
             custom_note = " Custom script will run on the Hub Job."
+        if j and j.eval_conditions:
+            cond_note = f" Evaluating: {', '.join(j.eval_conditions)} (others unchanged on success)."
     return True, (
         f"Approved job {job_id}. Up to {max_n} Hub Jobs may run in parallel; "
-        f"this job starts when a slot is free.{custom_note}"
+        f"this job starts when a slot is free.{custom_note}{cond_note}"
     )
 
 
@@ -1121,7 +1208,12 @@ def _job_can_retry(status: JobStatus) -> bool:
     return status in (JobStatus.failed, JobStatus.done, JobStatus.queued)
 
 
-def retry_failed_job(job_id: str, secret: str) -> tuple[bool, str]:
+def retry_failed_job(
+    job_id: str,
+    secret: str,
+    *,
+    eval_conditions: list[str] | None = None,
+) -> tuple[bool, str]:
     """Re-queue a job for another evaluation run (moderator only).
 
     Allowed for failed, done, and queued jobs. Successful re-runs replace the
@@ -1134,6 +1226,11 @@ def retry_failed_job(job_id: str, secret: str) -> tuple[bool, str]:
     if not job_id:
         return False, "Select a job."
 
+    try:
+        conds = normalize_moderator_eval_conditions(eval_conditions)
+    except ValueError as e:
+        return False, str(e)
+
     with _jobs_lock:
         job = _jobs.get(job_id)
         if not job:
@@ -1146,7 +1243,6 @@ def retry_failed_job(job_id: str, secret: str) -> tuple[bool, str]:
             )
         if _work_queue.qsize() >= _MAX_QUEUE_BACKLOG:
             return False, "Execution queue is full; try again later."
-
         was_queued = job.status == JobStatus.queued
         job.status = JobStatus.queued
         job.error = None
@@ -1156,17 +1252,27 @@ def retry_failed_job(job_id: str, secret: str) -> tuple[bool, str]:
         job.progress_done = 0
         job.progress_total = 0
         job.progress_condition = ""
-        job.replace_leaderboard = True
+        job.eval_conditions = conds
+        job.replace_leaderboard = conds is None
         _touch(job)
 
     if not was_queued:
         _work_queue.put(job_id)
     _persist_jobs()
     _ensure_worker()
+    cond_note = ""
+    with _jobs_lock:
+        j = _jobs.get(job_id)
+        if j and j.eval_conditions:
+            cond_note = (
+                f" Only {', '.join(j.eval_conditions)} will run; "
+                "existing leaderboard WER columns for other datasets are kept."
+            )
+        elif j and j.replace_leaderboard:
+            cond_note = " Full re-eval; the leaderboard row will be replaced on success."
     return (
         True,
-        f"Re-queued job {job_id}; it will run after jobs ahead of it. "
-        "If the model is already on the leaderboard, the row will be replaced on success.",
+        f"Re-queued job {job_id}; it will run after jobs ahead of it.{cond_note}"
     )
 
 

remote_jobs.py

@@ -153,6 +153,7 @@ def submit_eval_job(
     extra_requirements: str = "",
     custom_script: str = "",
     run_custom_script: bool = False,
+    eval_conditions: tuple[str, ...] | None = None,
 ) -> JobInfo:
     """Start a Hub UV Job; returns initial ``JobInfo`` (may already be RUNNING)."""
     namespace = os.environ.get("FFASR_REMOTE_JOB_NAMESPACE", "").strip() or None
@@ -225,6 +226,10 @@ def submit_eval_job(
         env["FFASR_CUSTOM_SCRIPT_B64"] = base64.b64encode(
             custom_script.strip().encode("utf-8")
         ).decode("ascii")
+    if eval_conditions:
+        import json
+
+        env["FFASR_EVAL_CONDITIONS"] = json.dumps(list(eval_conditions))
     secrets = {"HF_TOKEN": bucket_tok}
 
     api = HfApi(token=jobs_tok)

scripts/run_hf_remote_job_uv.py

@@ -119,11 +119,16 @@ def main() -> int:
         custom_script = (
             base64.b64decode(custom_b64).decode("utf-8") if custom_b64 else ""
         )
+        cond_raw = os.environ.get("FFASR_EVAL_CONDITIONS", "").strip()
+        condition_keys = None
+        if cond_raw:
+            condition_keys = json.loads(cond_raw)
         result = run_evaluation(
             model_id,
             family_id=family_id,
             progress_cb=None,
             custom_script=custom_script,
+            condition_keys=condition_keys,
         )
         artifact = build_artifact(
             model_id=model_id,