retry done jobs too

app.py

@@ -519,7 +519,9 @@ with gr.Blocks(title=APP_TITLE, theme=_theme, css=LEADERBOARD_CSS) as demo:
                     "When <code>FFASR_MODERATION=1</code> is set, new submissions **wait here** until you approve them. "
                     "Privileged actions require the same secret you used to unlock this panel.\n\n"
                     "When <code>FFASR_REMOTE_JOBS=1</code>, use **Open Hub Job logs** while a job runs. "
-                    "The bucket <code>results/remote_artifacts/</code> folder only gets a JSON file **after** a successful run."
+                    "The bucket <code>results/remote_artifacts/</code> folder only gets a JSON file **after** a successful run.\n\n"
+                    "**Retry** re-queues failed, done, or queued jobs (not while running). "
+                    "A successful re-run **replaces** the leaderboard row for that model if it already exists."
                 )
                 gr.Markdown("### Current job progress")
                 mod_progress = gr.HTML(

job_queue.py

@@ -72,6 +72,8 @@ class Job:
     progress_condition: str = ""
     hf_remote_job_id: str | None = None
     remote_artifact_path: str | None = None
+    # Set by moderator retry: skip duplicate guard and replace CSV row on success.
+    replace_leaderboard: bool = False
 
 
 _jobs: dict[str, Job] = {}
@@ -481,18 +483,35 @@ def _succeed_job(job_id: str, result: dict) -> None:
 
 
 def _merge_eval_result_to_leaderboard(
-    result: dict, submitted_at_iso: str, submission_notes: str
+    result: dict,
+    submitted_at_iso: str,
+    submission_notes: str,
+    *,
+    replace_existing: bool = False,
 ) -> None:
-    from init import leaderboard_row_from_eval_result, load_raw_results, save_raw_results
+    from init import (
+        invalidate_results_cache,
+        leaderboard_row_from_eval_result,
+        load_raw_results,
+        normalize_legacy_csv_row,
+        save_raw_results,
+    )
 
+    model_id = str(result.get("model_id", "")).strip()
     rows = load_raw_results()
-    rows.append(
-        leaderboard_row_from_eval_result(
-            result, submitted_at_iso, submission_notes=submission_notes
-        )
+    if replace_existing and model_id:
+        existing = [i for i, r in enumerate(rows) if (r.get("model_id") or "").strip() == model_id]
+        for i in sorted(existing, reverse=True):
+            rows.pop(i)
+
+    new_row = leaderboard_row_from_eval_result(
+        result, submitted_at_iso, submission_notes=submission_notes
     )
+    normalize_legacy_csv_row(new_row)
+    rows.append(new_row)
     _leaderboard_sort_rows_inplace(rows)
     save_raw_results(rows)
+    invalidate_results_cache()
 
 
 def _load_artifact_json_from_bucket(artifact_path: str) -> dict:
@@ -754,7 +773,10 @@ def _remote_start_queued_job(job_id: str, jobs_token: str) -> None:
     _persist_jobs()
 
     rows = load_raw_results()
-    if any(r["model_id"] == mid for r in rows):
+    with _jobs_lock:
+        j_chk = _jobs.get(job_id)
+        replace_lb = bool(j_chk and j_chk.replace_leaderboard)
+    if not replace_lb and any(r["model_id"] == mid for r in rows):
         _fail_job(job_id, "Model already on leaderboard (skipped duplicate race).")
         return
 
@@ -811,7 +833,16 @@ def _remote_tick_job(job_id: str, jobs_token: str) -> None:
 
     try:
         result = _remote_collect_result(job_id, hf_id, jobs_token)
-        _merge_eval_result_to_leaderboard(result, _now_iso(), notes)
+        with _jobs_lock:
+            j_done = _jobs.get(job_id)
+            replace_lb = bool(j_done and j_done.replace_leaderboard)
+        _merge_eval_result_to_leaderboard(
+            result, _now_iso(), notes, replace_existing=replace_lb
+        )
+        with _jobs_lock:
+            j_clr = _jobs.get(job_id)
+            if j_clr:
+                j_clr.replace_leaderboard = False
         _succeed_job(job_id, result)
     except Exception as e:
         _fail_job(job_id, str(e))
@@ -1077,8 +1108,25 @@ def moderation_locked_placeholder_html() -> str:
     return "<p><em>Moderator tools are locked. Enter the secret above and click Unlock.</em></p>"
 
 
+def _job_can_retry(status: JobStatus) -> bool:
+    """Moderator may re-run jobs that are not actively executing on Hub."""
+    if status in (
+        JobStatus.running,
+        JobStatus.dispatching,
+        JobStatus.remote_running,
+        JobStatus.collecting,
+        JobStatus.pending_moderation,
+    ):
+        return False
+    return status in (JobStatus.failed, JobStatus.done, JobStatus.queued)
+
+
 def retry_failed_job(job_id: str, secret: str) -> tuple[bool, str]:
-    """Re-queue a failed job for another evaluation run (moderator only)."""
+    """Re-queue a job for another evaluation run (moderator only).
+
+    Allowed for failed, done, and queued jobs. Successful re-runs replace the
+    existing leaderboard row for that model when one is present.
+    """
     ok, msg = _moderator_secret_ok(secret)
     if not ok:
         return False, msg
@@ -1090,13 +1138,16 @@ def retry_failed_job(job_id: str, secret: str) -> tuple[bool, str]:
         job = _jobs.get(job_id)
         if not job:
             return False, "Job not found."
-        if job.status == JobStatus.running:
-            return False, "Job is currently running."
-        if job.status != JobStatus.failed:
-            return False, "Only failed jobs can be retried."
+        if not _job_can_retry(job.status):
+            return (
+                False,
+                "Cannot retry while the job is running or awaiting moderation. "
+                "Wait for it to finish, or approve/reject pending jobs first.",
+            )
         if _work_queue.qsize() >= _MAX_QUEUE_BACKLOG:
             return False, "Execution queue is full; try again later."
 
+        was_queued = job.status == JobStatus.queued
         job.status = JobStatus.queued
         job.error = None
         job.result = None
@@ -1105,12 +1156,18 @@ def retry_failed_job(job_id: str, secret: str) -> tuple[bool, str]:
         job.progress_done = 0
         job.progress_total = 0
         job.progress_condition = ""
+        job.replace_leaderboard = True
         _touch(job)
 
-    _work_queue.put(job_id)
+    if not was_queued:
+        _work_queue.put(job_id)
     _persist_jobs()
     _ensure_worker()
-    return True, f"Re-queued job {job_id}; it will run after jobs ahead of it."
+    return (
+        True,
+        f"Re-queued job {job_id}; it will run after jobs ahead of it. "
+        "If the model is already on the leaderboard, the row will be replaced on success.",
+    )
 
 
 def remove_job_entry(job_id: str, secret: str) -> tuple[bool, str]:
@@ -1317,7 +1374,7 @@ def recent_jobs_for_render(limit: int = 30) -> list[dict[str, Any]]:
                 "error": err[:200] + ("…" if len(err) > 200 else ""),
                 "hub_link_html": hub,
                 "updated_at": (j.updated_at or j.created_at)[:19],
-                "can_retry": j.status == JobStatus.failed,
+                "can_retry": _job_can_retry(j.status),
                 "can_remove": j.status != JobStatus.running,
                 "has_custom_script": bool((j.custom_script or "").strip()),
             }