Sync src/ to 86e2b67 (viewer XSS fix + judge hardening)

src/ocr_bench/backends.py

@@ -5,19 +5,57 @@ from __future__ import annotations
 import abc
 from collections import Counter
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from itertools import zip_longest
 from typing import Any
 
 import stamina
 import structlog
 from huggingface_hub import InferenceClient
-from openai import OpenAI
+from openai import APIConnectionError, OpenAI
 
 from ocr_bench.judge import JUDGE_SCHEMA, Comparison, parse_judge_output
 
 logger = structlog.get_logger()
 
-# Retry on these exception types with exponential backoff + jitter.
-_RETRYABLE = (Exception,)
+# Number of attempts for a transient judge call (connection/timeout/429/5xx).
+_RETRY_ATTEMPTS = 5
+
+# Transport-level failures worth retrying — no HTTP status (connection reset,
+# DNS failure, read timeout, server hang-up mid-response). `APIConnectionError`
+# also covers the openai client's `APITimeoutError` (a subclass). HTTP errors are
+# handled by status code below.
+_TRANSPORT_ERRORS: tuple[type[BaseException], ...] = (APIConnectionError,)
+try:  # requests is always present via huggingface_hub; guard defensively
+    from requests.exceptions import ConnectionError as _ReqConnectionError
+    from requests.exceptions import Timeout as _ReqTimeout
+
+    _TRANSPORT_ERRORS += (_ReqConnectionError, _ReqTimeout)
+except Exception:  # pragma: no cover
+    pass
+try:  # httpx backs huggingface_hub's InferenceClient — the default judge path.
+    # Providers (e.g. Novita) disconnect on long runs, raising httpx
+    # TransportError subclasses (RemoteProtocolError, ConnectError, timeouts).
+    from httpx import TransportError as _HttpxTransportError
+
+    _TRANSPORT_ERRORS += (_HttpxTransportError,)
+except Exception:  # pragma: no cover
+    pass
+
+
+def _is_retryable(exc: Exception) -> bool:
+    """Retry only *transient* judge failures.
+
+    Retries connection/timeout errors and HTTP 429 (rate limit) or 5xx
+    (server) responses. Fatal errors — a bad/expired token, a typo'd model
+    id, or any other 4xx — are NOT retried, so they surface immediately
+    instead of after several pointless backoffs.
+    """
+    status = getattr(exc, "status_code", None)
+    if status is None:
+        status = getattr(getattr(exc, "response", None), "status_code", None)
+    if status is not None:
+        return status == 429 or status >= 500
+    return isinstance(exc, _TRANSPORT_ERRORS)
 
 
 class JudgeBackend(abc.ABC):
@@ -38,7 +76,14 @@ class JudgeBackend(abc.ABC):
         or an empty dict on failure.
         """
         if self.concurrency <= 1 or len(comparisons) <= 1:
-            return [self._call_single(comp) for comp in comparisons]
+            results = []
+            for i, comp in enumerate(comparisons):
+                try:
+                    results.append(self._call_single(comp))
+                except Exception as exc:
+                    logger.warning("judge_call_failed", idx=i, error=str(exc))
+                    results.append({})
+            return results
 
         # Concurrent execution preserving order
         results: list[dict[str, str]] = [{}] * len(comparisons)
@@ -69,11 +114,11 @@ class InferenceProviderJudge(JudgeBackend):
         self.name = f"{provider + ':' if provider else ''}{model}"
         self.model = model
         self.max_tokens = max_tokens
-        self.client = InferenceClient(model=model, provider=provider)  # type: ignore[invalid-argument-type]
+        self.client = InferenceClient(model=model, provider=provider)  # ty: ignore[invalid-argument-type]
 
-    @stamina.retry(on=_RETRYABLE, attempts=6)
+    @stamina.retry(on=_is_retryable, attempts=_RETRY_ATTEMPTS)
     def _call_single(self, comp: Comparison) -> dict[str, str]:
-        response = self.client.chat_completion(  # type: ignore[no-matching-overload]
+        response = self.client.chat_completion(  # ty: ignore[no-matching-overload]
             messages=comp.messages,
             max_tokens=self.max_tokens,
             temperature=0.0,
@@ -108,11 +153,11 @@ class OpenAICompatibleJudge(JudgeBackend):
         self.concurrency = concurrency
         self.client = OpenAI(base_url=base_url, api_key=api_key)
 
-    @stamina.retry(on=_RETRYABLE, attempts=3)
+    @stamina.retry(on=_is_retryable, attempts=_RETRY_ATTEMPTS)
     def _call_single(self, comp: Comparison) -> dict[str, str]:
         response = self.client.chat.completions.create(
             model=self.model,
-            messages=comp.messages,  # type: ignore[invalid-argument-type]
+            messages=comp.messages,  # ty: ignore[invalid-argument-type]
             max_tokens=self.max_tokens,
             temperature=self.temperature,
             extra_body=self.extra_body,
@@ -128,7 +173,7 @@ class OpenAICompatibleJudge(JudgeBackend):
 # Spec parsing
 # ---------------------------------------------------------------------------
 
-DEFAULT_JUDGE = "novita:moonshotai/Kimi-K2.5"
+DEFAULT_JUDGE = "novita:Qwen/Qwen3.5-35B-A3B"
 
 
 def parse_judge_spec(
@@ -165,7 +210,7 @@ def parse_judge_spec(
                 model=model_name,
                 api_key=token,
                 max_tokens=max_tokens,
-                temperature=0.7,
+                temperature=0.0,
                 extra_body={"chat_template_kwargs": {"enable_thinking": False}},
                 concurrency=concurrency,
             )
@@ -207,26 +252,31 @@ def aggregate_jury_votes(
     if not all_results:
         return []
 
-    n_comparisons = len(all_results[0])
-    n_judges = len(all_results)
     aggregated: list[dict[str, Any]] = []
 
-    for i in range(n_comparisons):
+    # Transpose judge-major results to comparison-major; a judge that
+    # returned a short list pads out with failures.
+    for judge_results in zip_longest(*all_results, fillvalue={}):
         votes: list[str] = []
         reasons: list[str] = []
-        for j in range(n_judges):
-            result = all_results[j][i] if i < len(all_results[j]) else {}
+        for name, result in zip(judge_names, judge_results):
             winner = result.get("winner", "")
             if winner:
                 votes.append(winner)
-                reasons.append(f"{judge_names[j]}: {result.get('reason', '')}")
+                reasons.append(f"{name}: {result.get('reason', '')}")
 
         if not votes:
             aggregated.append({"winner": "tie", "reason": "no valid votes", "agreement": "0/0"})
             continue
 
         counter = Counter(votes)
-        majority_winner, majority_count = counter.most_common(1)[0]
+        top = counter.most_common(2)
+        majority_winner, majority_count = top[0]
+        if len(top) > 1 and top[1][1] == majority_count:
+            # No strict majority (1-1, 2-2, three-way split, ...): record a
+            # tie rather than letting Counter's insertion order side with
+            # whichever judge happens to be listed first.
+            majority_winner = "tie"
         agreement = f"{majority_count}/{len(votes)}"
 
         aggregated.append({

src/ocr_bench/cli.py

@@ -6,6 +6,7 @@ import argparse
 import sys
 
 import structlog
+from openai import OpenAIError
 from rich.console import Console
 from rich.table import Table
 
@@ -87,6 +88,15 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Don't publish results (default: publish to {dataset}-results)",
     )
+    judge.add_argument(
+        "--license",
+        default=None,
+        help=(
+            "License tag for the published results dataset card, e.g. cc0-1.0 "
+            "(default: none — the results embed source-derived text, so only "
+            "the publisher knows the right license)"
+        ),
+    )
     judge.add_argument(
         "--full-rejudge",
         action="store_true",
@@ -132,14 +142,26 @@ def build_parser() -> argparse.ArgumentParser:
     view.add_argument("--host", default="127.0.0.1", help="Host (default: 127.0.0.1)")
     view.add_argument("--output", default=None, help="Path to save annotations JSON")
 
+    # --- publish subcommand ---
+    publish = sub.add_parser("publish", help="Deploy results viewer as a Hugging Face Space")
+    publish.add_argument("results", help="HF results dataset repo id to view in the Space")
+    publish.add_argument(
+        "--space", default=None, help="Space repo id (default: {results}-viewer)"
+    )
+    publish.add_argument("--private", action="store_true", help="Make the Space private")
+
     return parser
 
 
 def print_leaderboard(board: Leaderboard) -> None:
     """Print leaderboard as a Rich table."""
+    from ocr_bench.publish import _get_model_sizes
+
+    sizes = _get_model_sizes()
     table = Table(title="OCR Model Leaderboard")
     table.add_column("Rank", style="bold")
     table.add_column("Model")
+    table.add_column("Params", justify="right")
     has_ci = bool(board.elo_ci)
     if has_ci:
         table.add_column("ELO (95% CI)", justify="right")
@@ -161,6 +183,7 @@ def print_leaderboard(board: Leaderboard) -> None:
         table.add_row(
             str(rank),
             model,
+            sizes.get(model, ""),
             elo_str,
             str(board.wins[model]),
             str(board.losses[model]),
@@ -177,7 +200,9 @@ def _convert_results(
     """Convert judged comparisons + aggregated outputs into ComparisonResult list."""
     results: list[ComparisonResult] = []
     for comp, result in zip(comparisons, aggregated):
-        if not result:
+        # Skip failures: empty dict (single judge failed) and 0/0 "ties"
+        # (every judge in a jury failed) — neither is a real verdict.
+        if not result or result.get("agreement") == "0/0":
             continue
         results.append(
             ComparisonResult(
@@ -420,6 +445,7 @@ def cmd_judge(args: argparse.Namespace) -> None:
                     board,
                     metadata,
                     existing_metadata=existing_meta_rows,
+                    license_id=args.license,
                 )
                 console.print(f"\nResults published to [bold]{results_repo}[/bold]")
             return
@@ -451,7 +477,13 @@ def cmd_judge(args: argparse.Namespace) -> None:
             valid_comparisons=len(new_results),
             from_prs=from_prs,
         )
-        publish_results(results_repo, board, metadata, existing_metadata=existing_meta_rows)
+        publish_results(
+            results_repo,
+            board,
+            metadata,
+            existing_metadata=existing_meta_rows,
+            license_id=args.license,
+        )
         console.print(f"\nResults published to [bold]{results_repo}[/bold]")
 
 
@@ -476,7 +508,8 @@ def cmd_run(args: argparse.Namespace) -> None:
         for slug in sorted(MODEL_REGISTRY):
             cfg = MODEL_REGISTRY[slug]
             default = " (default)" if slug in DEFAULT_MODELS else ""
-            table.add_row(slug + default, cfg.model_id, cfg.size, cfg.default_flavor)
+            gpu = cfg.default_flavor + (" (image-mode)" if cfg.image else "")
+            table.add_row(slug + default, cfg.model_id, cfg.size, gpu)
 
         console.print(table)
         console.print(f"\nDefault set: {', '.join(DEFAULT_MODELS)}")
@@ -515,6 +548,10 @@ def cmd_run(args: argparse.Namespace) -> None:
             console.print(f"[cyan]{slug}[/cyan] ({cfg.model_id})")
             console.print(f"  Flavor:  {flavor}")
             console.print(f"  Timeout: {args.timeout}")
+            if cfg.image:
+                console.print(f"  Image:   {cfg.image}")
+                console.print(f"  Python:  {cfg.python}")
+                console.print(f"  Env:     {cfg.env}")
             console.print(f"  Script:  {cfg.script}")
             console.print(f"  Args:    {' '.join(script_args)}")
             console.print()
@@ -569,6 +606,47 @@ def cmd_view(args: argparse.Namespace) -> None:
     uvicorn.run(app, host=args.host, port=args.port)
 
 
+SPACE_TEMPLATE = "davanstrien/ocr-bench-space-template"
+
+
+def cmd_publish(args: argparse.Namespace) -> None:
+    """Deploy results viewer as a Hugging Face Space."""
+    from huggingface_hub import HfApi, SpaceHardware
+
+    api = HfApi()
+    results = args.results
+    space_id = args.space or f"{results}-viewer"
+
+    console.print(f"Deploying viewer for [bold]{results}[/bold] to [bold]{space_id}[/bold]...")
+
+    api.duplicate_space(
+        from_id=SPACE_TEMPLATE,
+        to_id=space_id,
+        private=args.private if args.private else None,
+        hardware=SpaceHardware.CPU_BASIC,
+        exist_ok=True,
+        variables=[{"key": "REPOS", "value": results}],
+    )
+
+    api.add_space_variable(repo_id=space_id, key="REPOS", value=results)
+
+    # Update Space metadata to link to results dataset
+    try:
+        from huggingface_hub import metadata_update
+
+        metadata_update(
+            space_id,
+            {"datasets": [results], "tags": ["ocr-bench"]},
+            repo_type="space",
+            overwrite=True,
+        )
+    except Exception as exc:
+        logger.warning("space_metadata_update_failed", error=str(exc))
+
+    url = f"https://huggingface.co/spaces/{space_id}"
+    console.print(f"[green]Space published![/green] {url}")
+
+
 def main() -> None:
     parser = build_parser()
     args = parser.parse_args()
@@ -584,6 +662,15 @@ def main() -> None:
             cmd_run(args)
         elif args.command == "view":
             cmd_view(args)
+        elif args.command == "publish":
+            cmd_publish(args)
     except DatasetError as exc:
         console.print(f"[red]Error:[/red] {exc}")
         sys.exit(1)
+    except (OpenAIError, OSError) as exc:
+        # Judge or Hub request failed (bad/expired token, unknown model id,
+        # rate limit, provider/network outage) — every requests/HfHubHTTPError
+        # subclasses OSError — or another OS-level error. Fail with a clean
+        # message instead of dumping a traceback on the user.
+        console.print(f"[red]Error:[/red] {exc}")
+        sys.exit(1)

src/ocr_bench/dataset.py

@@ -211,11 +211,46 @@ def load_config_dataset(
     if unified is None:
         raise DatasetError("No configs loaded successfully")
 
+    # Disambiguate configs that resolve to the same model_id (mirrors the flat
+    # discover_ocr_columns path). Downstream the ELO keys models by these values,
+    # so two configs of the same model run with different settings (e.g.
+    # `nuextract3` vs `nuextract3-rep`) would silently collapse into one
+    # leaderboard row. On collision, label by config name; keep the bare
+    # model_id when unique.
+    model_counts: dict[str, int] = {}
+    for model_id in ocr_columns.values():
+        model_counts[model_id] = model_counts.get(model_id, 0) + 1
+    duplicates = sorted(mid for mid, n in model_counts.items() if n > 1)
+    if duplicates:
+        # Capture the colliding config → model_id mapping before relabeling so
+        # the warning is actionable (which configs/model_ids collided, in which
+        # repo) when running multiple datasets/config sweeps.
+        collided = {
+            config: model_id
+            for config, model_id in ocr_columns.items()
+            if model_counts[model_id] > 1
+        }
+        for config, model_id in list(ocr_columns.items()):
+            if model_counts[model_id] > 1:
+                short = model_id.split("/")[-1] if "/" in model_id else model_id
+                ocr_columns[config] = f"{short} ({config})"
+        logger.warning(
+            "duplicate_model_ids",
+            repo_id=repo_id,
+            model_ids=duplicates,
+            collided_configs=collided,
+            note="configs sharing a model_id were labelled by config name to keep them distinct",
+        )
+
     return unified, ocr_columns
 
 
 def _extract_model_id(ds: Dataset, config: str) -> str:
-    """Extract model_id from inference_info in first row, falling back to config name."""
+    """Extract model_id from inference_info in first row, falling back to config name.
+
+    Takes the *last* entry in the inference_info list, since OCR scripts append
+    new entries — the last one is the model that actually produced this config.
+    """
     if "inference_info" not in ds.column_names:
         return config
     try:
@@ -223,7 +258,7 @@ def _extract_model_id(ds: Dataset, config: str) -> str:
         if info_raw:
             info = json.loads(info_raw)
             if isinstance(info, list):
-                info = info[0]
+                info = info[-1]
             return info.get("model_id", info.get("model_name", config))
     except (json.JSONDecodeError, TypeError, KeyError, IndexError):
         pass

src/ocr_bench/elo.py

@@ -282,13 +282,24 @@ def compute_elo(
             board.ties[r.model_a] += 1
             board.ties[r.model_b] += 1
 
+        # Canonicalise the reason text so A/B references match model_a/model_b
+        reason = r.reason
+        if r.swapped and reason:
+            # Swap "Output A"↔"Output B" (and bare A/B) so the stored reason
+            # uses A/B consistently with model_a/model_b ordering.
+            reason = (
+                reason.replace("Output A", "Output __X__")
+                .replace("Output B", "Output A")
+                .replace("Output __X__", "Output B")
+            )
+
         board.comparison_log.append(
             {
                 "sample_idx": r.sample_idx,
                 "model_a": r.model_a,
                 "model_b": r.model_b,
                 "winner": winner,
-                "reason": r.reason,
+                "reason": reason,
                 "agreement": r.agreement,
                 "text_a": r.text_a,
                 "text_b": r.text_b,

src/ocr_bench/judge.py

@@ -40,8 +40,10 @@ stamps, handwritten notes. Missing any section of text is a significant penalty.
 4. Reading order: Text flows naturally as a human would read the document.
 
 5. Formatting: Clean structure. Ignore bounding box tags like <|ref|> <|det|> \
-if present. Do NOT prefer fancier markdown formatting — plain accurate text is \
-better than nicely formatted but incomplete text.
+if present. Markdown formatting markers (#, **, *, etc.) are neutral — do not \
+penalise or reward their presence. Judge only on the actual text content, not \
+on whether it is wrapped in markup. Plain text and markdown-formatted text that \
+contain the same words are equivalent.
 
 If both outputs capture the same text with similar accuracy, respond with "tie". \
 Only pick a winner when there is a clear quality difference.
@@ -273,15 +275,27 @@ def parse_judge_output(text: str) -> dict[str, str]:
     """
     text = text.strip()
     if text.startswith("```"):
-        text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
+        # A truncated response can be a bare opening fence with no body
+        parts = text.split("\n", 1)
+        text = parts[1].rsplit("```", 1)[0].strip() if len(parts) == 2 else ""
     try:
         result = json.loads(text)
-        winner = result.get("winner", "tie").upper().strip()
-        if winner == "TIE":
-            winner = "tie"
-        if winner not in ("A", "B", "tie"):
-            winner = "tie"
-        return {"winner": winner, "reason": result.get("reason", "")}
     except json.JSONDecodeError:
         logger.warning("Failed to parse judge output: %s", text[:200])
         return {}
+    if not isinstance(result, dict):
+        logger.warning("Judge output is not a JSON object: %s", text[:200])
+        return {}
+    winner = result.get("winner", "tie")
+    if not isinstance(winner, str):
+        logger.warning("Judge output has non-string winner: %s", text[:200])
+        return {}
+    winner = winner.upper().strip()
+    if winner == "TIE":
+        winner = "tie"
+    if winner not in ("A", "B", "tie"):
+        winner = "tie"
+    reason = result.get("reason", "")
+    if not isinstance(reason, str):
+        reason = json.dumps(reason)
+    return {"winner": winner, "reason": reason}

src/ocr_bench/publish.py

@@ -11,6 +11,7 @@ from datasets import Dataset, load_dataset
 from huggingface_hub import HfApi
 
 from ocr_bench.elo import ComparisonResult, Leaderboard
+from ocr_bench.run import MODEL_REGISTRY
 
 logger = structlog.get_logger()
 
@@ -79,14 +80,21 @@ def load_existing_metadata(repo_id: str) -> list[dict]:
         return []
 
 
+def _get_model_sizes() -> dict[str, str]:
+    """Build model_id → size lookup from the model registry."""
+    return {cfg.model_id: cfg.size for cfg in MODEL_REGISTRY.values()}
+
+
 def build_leaderboard_rows(board: Leaderboard) -> list[dict]:
     """Convert a Leaderboard into rows suitable for a Hub dataset."""
+    sizes = _get_model_sizes()
     rows = []
     for model, elo in board.ranked:
         total = board.wins[model] + board.losses[model] + board.ties[model]
         row = {
             "model": model,
             "elo": round(elo),
+            "params": sizes.get(model, ""),
             "wins": board.wins[model],
             "losses": board.losses[model],
             "ties": board.ties[model],
@@ -119,6 +127,7 @@ def publish_results(
     board: Leaderboard,
     metadata: EvalMetadata,
     existing_metadata: list[dict] | None = None,
+    license_id: str | None = None,
 ) -> None:
     """Push evaluation results to Hub as a dataset with multiple configs.
 
@@ -151,7 +160,7 @@ def publish_results(
     logger.info("published_metadata", repo=repo_id, n=len(all_meta))
 
     # README — auto-generated dataset card with leaderboard
-    readme = _build_readme(repo_id, rows, board, metadata)
+    readme = _build_readme(repo_id, rows, board, metadata, license_id=license_id)
     api = HfApi()
     api.upload_file(
         path_or_fileobj=readme.encode(),
@@ -167,6 +176,7 @@ def _build_readme(
     rows: list[dict],
     board: Leaderboard,
     metadata: EvalMetadata,
+    license_id: str | None = None,
 ) -> str:
     """Build a dataset card README with the leaderboard table."""
     has_ci = bool(board.elo_ci)
@@ -179,12 +189,19 @@ def _build_readme(
     judge_str = ", ".join(j.split("/")[-1] for j in judges) if judges else "N/A"
     n_comparisons = len(board.comparison_log)
 
-    lines = [
-        "---",
-        "license: mit",
+    # The card license describes the published results DATA (which embeds
+    # OCR text derived from the source dataset), not this tool — so there is
+    # no correct default; it's declared per-run via --license or set on the
+    # Hub repo by the publisher.
+    lines = ["---"]
+    if license_id:
+        lines.append(f"license: {license_id}")
+    lines += [
         "tags:",
         "  - ocr-bench",
         "  - leaderboard",
+        "source_datasets:",
+        f"  - {metadata.source_dataset}",
         "configs:",
         "  - config_name: default",
         "    data_files:",
@@ -215,25 +232,27 @@ def _build_readme(
 
     # Table header
     if has_ci:
-        lines.append("| Rank | Model | ELO | 95% CI | Wins | Losses | Ties | Win% |")
-        lines.append("|------|-------|-----|--------|------|--------|------|------|")
+        lines.append("| Rank | Model | Params | ELO | 95% CI | Wins | Losses | Ties | Win% |")
+        lines.append("|------|-------|--------|-----|--------|------|--------|------|------|")
     else:
-        lines.append("| Rank | Model | ELO | Wins | Losses | Ties | Win% |")
-        lines.append("|------|-------|-----|------|--------|------|------|")
+        lines.append("| Rank | Model | Params | ELO | Wins | Losses | Ties | Win% |")
+        lines.append("|------|-------|--------|-----|------|--------|------|------|")
 
     for rank, row in enumerate(rows, 1):
-        model = row["model"]
+        # Escape pipes so arbitrary model names can't break the table
+        model = str(row["model"]).replace("|", "\\|")
         elo = row["elo"]
+        params = row.get("params", "")
         if has_ci and "elo_low" in row:
             ci = f"{row['elo_low']}\u2013{row['elo_high']}"
             lines.append(
-                f"| {rank} | {model} | {elo} | {ci} "
+                f"| {rank} | {model} | {params} | {elo} | {ci} "
                 f"| {row['wins']} | {row['losses']} | {row['ties']} "
                 f"| {row['win_pct']}% |"
             )
         else:
             lines.append(
-                f"| {rank} | {model} | {elo} "
+                f"| {rank} | {model} | {params} | {elo} "
                 f"| {row['wins']} | {row['losses']} | {row['ties']} "
                 f"| {row['win_pct']}% |"
             )

src/ocr_bench/run.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import time
 from dataclasses import dataclass, field
+from typing import Any
 
 import structlog
 from huggingface_hub import HfApi, get_token
@@ -13,13 +14,33 @@ logger = structlog.get_logger()
 
 @dataclass
 class ModelConfig:
-    """Configuration for a single OCR model."""
+    """Configuration for a single OCR model.
+
+    ``image`` / ``python`` / ``env`` are only needed by "image-mode" models —
+    ones whose CUDA kernels (e.g. flashinfer for Qwen3.5) must come from a
+    prebuilt Docker image because the default uv-script image lacks ``nvcc``.
+    They are passed straight through to ``run_uv_job`` and left ``None`` for
+    every standard model, which keeps the launch call identical to before.
+    """
 
     script: str
     model_id: str
     size: str
     default_flavor: str = "l4x1"
     default_args: list[str] = field(default_factory=list)
+    image: str | None = None
+    python: str | None = None
+    env: dict[str, str] | None = None
+
+
+# Image-mode invocation for models needing prebuilt CUDA kernels (Qwen3.5 /
+# flashinfer). The default uv-script image has no ``nvcc`` so flashinfer's JIT
+# compile fails at vLLM warmup; the vllm/vllm-openai image ships them prebuilt.
+# ``python`` points at that image's interpreter and ``env`` puts its site-packages
+# on the path so ``uv run`` reuses them instead of rebuilding.
+_VLLM_OPENAI_IMAGE = "vllm/vllm-openai:latest"
+_VLLM_OPENAI_PYTHON = "/usr/bin/python3"
+_VLLM_OPENAI_ENV = {"PYTHONPATH": "/usr/local/lib/python3.12/dist-packages"}
 
 
 MODEL_REGISTRY: dict[str, ModelConfig] = {
@@ -48,9 +69,46 @@ MODEL_REGISTRY: dict[str, ModelConfig] = {
         size="1.7B",
         default_flavor="l4x1",
     ),
+    "firered-ocr": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/firered-ocr.py",
+        model_id="FireRedTeam/FireRed-OCR",
+        size="2.1B",
+        default_flavor="l4x1",
+    ),
+    "qianfan-ocr": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/qianfan-ocr.py",
+        model_id="baidu/Qianfan-OCR",
+        size="4.7B",
+        default_flavor="l4x1",
+    ),
+    "dots-mocr": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/dots-mocr.py",
+        model_id="rednote-hilab/dots.mocr",
+        size="3B",
+        default_flavor="l4x1",
+    ),
+    # Image-mode models (Qwen3.5 / flashinfer) — need the vllm/vllm-openai image.
+    "nuextract3": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/nuextract3.py",
+        model_id="numind/NuExtract3",
+        size="4B",
+        default_flavor="a100-large",
+        image=_VLLM_OPENAI_IMAGE,
+        python=_VLLM_OPENAI_PYTHON,
+        env=_VLLM_OPENAI_ENV,
+    ),
+    "paddleocr-vl-1.6": ModelConfig(
+        script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/paddleocr-vl-1.6.py",
+        model_id="PaddlePaddle/PaddleOCR-VL-1.6",
+        size="0.9B",
+        default_flavor="a100-large",
+        image=_VLLM_OPENAI_IMAGE,
+        python=_VLLM_OPENAI_PYTHON,
+        env=_VLLM_OPENAI_ENV,
+    ),
 }
 
-DEFAULT_MODELS = ["glm-ocr", "deepseek-ocr", "lighton-ocr-2", "dots-ocr"]
+DEFAULT_MODELS = ["glm-ocr", "deepseek-ocr", "lighton-ocr-2", "dots-ocr", "firered-ocr"]
 
 
 @dataclass
@@ -139,13 +197,30 @@ def launch_ocr_jobs(
             extra_args=config.default_args or None,
         )
 
-        logger.info("launching_job", model=slug, flavor=flavor, script=config.script)
+        # Only image-mode models set image/python/env; standard models keep the
+        # exact same run_uv_job call as before (no extra kwargs).
+        extra_kwargs: dict[str, Any] = {}
+        if config.image:
+            extra_kwargs["image"] = config.image
+        if config.python:
+            extra_kwargs["python"] = config.python
+        if config.env:
+            extra_kwargs["env"] = config.env
+
+        logger.info(
+            "launching_job",
+            model=slug,
+            flavor=flavor,
+            script=config.script,
+            image=config.image,
+        )
         job = api.run_uv_job(
             script=config.script,
             script_args=script_args,
             flavor=flavor,
             secrets={"HF_TOKEN": token},
             timeout=timeout,
+            **extra_kwargs,
         )
         jobs.append(JobRun(model_slug=slug, job_id=job.id, job_url=job.url))
         logger.info("job_launched", model=slug, job_id=job.id, url=job.url)

src/ocr_bench/templates/leaderboard.html

@@ -2,13 +2,21 @@
 {% block title %}Leaderboard — OCR Bench{% endblock %}
 {% block content %}
 <h2 style="font-size: 1.1rem; font-weight: 600; margin-bottom: 0.25rem;">Leaderboard</h2>
-<p style="font-size: 0.8rem; color: #888; margin-top: 0;">{{ repo_id }}</p>
+<p style="font-size: 0.8rem; color: #888; margin-top: 0;"><a href="https://huggingface.co/datasets/{{ repo_id }}" style="color: #888; text-decoration: underline;" target="_blank">{{ repo_id }}</a></p>
+
+<p style="font-size: 0.82rem; color: #999; line-height: 1.5; max-width: 48rem; margin: 0.75rem 0 1rem;">
+  Rankings are computed using <strong style="color: #bbb;">Bradley-Terry MLE</strong> from pairwise comparisons judged by a vision-language model.
+  The judge sees the original document image alongside two anonymised OCR outputs and picks the more faithful transcription.
+  Browse the <a href="/comparisons" style="color: #aaa;">comparisons</a> to see the evidence — and vote yourself to build a Human ELO column.
+  Human votes are stored locally for this session only and will reset when the server restarts.
+</p>
 
 <table>
   <thead>
     <tr>
       <th>#</th>
       <th>Model</th>
+      <th class="num">Params</th>
       <th class="num">Judge ELO</th>
       {% if has_ci %}<th class="num">95% CI</th>{% endif %}
       <th class="num">Wins</th>
@@ -26,6 +34,7 @@
     <tr>
       <td>{{ loop.index }}</td>
       <td class="model">{{ row.model_short }}</td>
+      <td class="num">{{ row.params if row.params else "—" }}</td>
       <td class="num">{{ row.elo }}</td>
       {% if has_ci %}<td class="num">{{ row.elo_low }}&ndash;{{ row.elo_high }}</td>{% endif %}
       <td class="num">{{ row.wins }}</td>
@@ -40,4 +49,134 @@
     {% endfor %}
   </tbody>
 </table>
+
+{% if chart_points|length >= 2 %}
+<h3 style="font-size: 0.95rem; font-weight: 600; margin-top: 2rem; margin-bottom: 0.5rem;">
+  ELO vs Parameter Count
+</h3>
+<p style="font-size: 0.78rem; color: #888; margin-top: 0; margin-bottom: 0.75rem;">
+  Smaller models can win on the right documents. Error bars show 95% confidence intervals.
+</p>
+
+<div style="max-width: 560px; position: relative;">
+  <canvas id="paramsChart"></canvas>
+</div>
+
+<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
+<script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-datalabels@2"></script>
+<script>
+(function() {
+  const points = {{ chart_points | tojson }};
+  const colors = ['#6fa8dc', '#93c47d', '#e06666', '#f6b26b', '#8e7cc3'];
+
+  const gridColor = 'rgba(255,255,255,0.1)';
+  const textColor = '#aaa';
+
+  const datasets = points.map((p, i) => ({
+    label: p.name,
+    data: [{
+      x: p.params,
+      y: p.elo,
+      elo_low: p.elo_low,
+      elo_high: p.elo_high,
+      win_pct: p.win_pct,
+    }],
+    backgroundColor: colors[i % colors.length],
+    borderColor: colors[i % colors.length],
+    pointRadius: 10,
+    pointHoverRadius: 13,
+  }));
+
+  // Custom plugin to draw error bars
+  const errorBarPlugin = {
+    id: 'errorBars',
+    afterDatasetsDraw(chart) {
+      const { ctx } = chart;
+      chart.data.datasets.forEach((ds, i) => {
+        const meta = chart.getDatasetMeta(i);
+        meta.data.forEach((point, j) => {
+          const d = ds.data[j];
+          if (d.elo_low == null || d.elo_high == null) return;
+          const xPx = point.x;
+          const yLo = chart.scales.y.getPixelForValue(d.elo_low);
+          const yHi = chart.scales.y.getPixelForValue(d.elo_high);
+          const capW = 5;
+          ctx.save();
+          ctx.strokeStyle = ds.borderColor;
+          ctx.lineWidth = 1.5;
+          ctx.globalAlpha = 0.6;
+          // Vertical line
+          ctx.beginPath();
+          ctx.moveTo(xPx, yLo);
+          ctx.lineTo(xPx, yHi);
+          ctx.stroke();
+          // Top cap
+          ctx.beginPath();
+          ctx.moveTo(xPx - capW, yHi);
+          ctx.lineTo(xPx + capW, yHi);
+          ctx.stroke();
+          // Bottom cap
+          ctx.beginPath();
+          ctx.moveTo(xPx - capW, yLo);
+          ctx.lineTo(xPx + capW, yLo);
+          ctx.stroke();
+          ctx.restore();
+        });
+      });
+    },
+  };
+
+  Chart.register(ChartDataLabels, errorBarPlugin);
+
+  new Chart(document.getElementById('paramsChart'), {
+    type: 'scatter',
+    data: { datasets },
+    options: {
+      responsive: true,
+      animation: { duration: 400 },
+      scales: {
+        x: {
+          title: { display: true, text: 'Parameters (B)', color: textColor },
+          grid: { color: gridColor },
+          ticks: { color: textColor, callback: v => v + 'B' },
+        },
+        y: {
+          title: { display: true, text: 'ELO', color: textColor },
+          grid: { color: gridColor },
+          ticks: { color: textColor },
+        },
+      },
+      plugins: {
+        legend: { display: false },
+        tooltip: {
+          callbacks: {
+            label: ctx => {
+              const d = ctx.raw;
+              let s = `${ctx.dataset.label}: ${d.x}B, ELO ${d.y}`;
+              if (d.elo_low != null) s += ` (${d.elo_low}\u2013${d.elo_high})`;
+              s += `, ${d.win_pct}% wins`;
+              return s;
+            },
+          },
+        },
+        datalabels: {
+          align: function(ctx) {
+            const d = ctx.dataset.data[0];
+            const maxX = Math.max(...points.map(p => p.params));
+            // Rightmost point: label left. Otherwise label right.
+            if (d.x >= maxX) return 'left';
+            return 'right';
+          },
+          anchor: 'center',
+          offset: 12,
+          color: textColor,
+          font: { size: 11 },
+          formatter: (val, ctx) => ctx.dataset.label,
+        },
+      },
+    },
+  });
+})();
+</script>
+{% endif %}
 {% endblock %}

src/ocr_bench/validate.py

@@ -97,9 +97,23 @@ def _interleave_by_sample(
     return result
 
 
+def _has_overlapping_cis(
+    model_a: str,
+    model_b: str,
+    ci_map: dict[str, tuple[float, float]],
+) -> bool:
+    """Check if two models have overlapping confidence intervals."""
+    if model_a not in ci_map or model_b not in ci_map:
+        return True  # assume overlapping if CI data missing for a model
+    a_low, a_high = ci_map[model_a]
+    b_low, b_high = ci_map[model_b]
+    return max(a_low, b_low) < min(a_high, b_high)
+
+
 def build_validation_comparisons(
     comparison_rows: list[dict[str, Any]],
     *,
+    leaderboard_rows: list[dict[str, Any]] | None = None,
     n: int | None = None,
     prioritize_splits: bool = True,
     seed: int = 42,
@@ -108,6 +122,9 @@ def build_validation_comparisons(
 
     Args:
         comparison_rows: Rows from the comparisons config of a results dataset.
+        leaderboard_rows: Leaderboard rows with elo_low/elo_high for focus-pairs.
+            When provided, comparisons between models with overlapping CIs are
+            prioritized (those are where human input can change the ranking).
         n: Max number of comparisons to include (None = all).
         prioritize_splits: Show split-jury cases first (most informative).
         seed: Random seed for position-bias randomization.
@@ -144,7 +161,41 @@ def build_validation_comparisons(
             )
         )
 
-    if prioritize_splits:
+    # Build CI map from leaderboard rows (if available and has CI data)
+    ci_map: dict[str, tuple[float, float]] = {}
+    if leaderboard_rows:
+        for row in leaderboard_rows:
+            model = row.get("model", "")
+            lo = row.get("elo_low")
+            hi = row.get("elo_high")
+            if model and lo is not None and hi is not None:
+                ci_map[model] = (lo, hi)
+
+    if prioritize_splits and ci_map:
+        # 4-tier priority: overlapping+split > overlapping+unanimous >
+        # resolved+split > resolved+unanimous
+        overlap_split: list[ValidationComparison] = []
+        overlap_unanimous: list[ValidationComparison] = []
+        resolved_split: list[ValidationComparison] = []
+        resolved_unanimous: list[ValidationComparison] = []
+        for c in comps:
+            overlapping = _has_overlapping_cis(c.model_a, c.model_b, ci_map)
+            split = _is_split_jury(c.agreement)
+            if overlapping and split:
+                overlap_split.append(c)
+            elif overlapping:
+                overlap_unanimous.append(c)
+            elif split:
+                resolved_split.append(c)
+            else:
+                resolved_unanimous.append(c)
+        ordered = (
+            _interleave_by_sample(overlap_split)
+            + _interleave_by_sample(overlap_unanimous)
+            + _interleave_by_sample(resolved_split)
+            + _interleave_by_sample(resolved_unanimous)
+        )
+    elif prioritize_splits:
         splits = [c for c in comps if _is_split_jury(c.agreement)]
         unanimous = [c for c in comps if not _is_split_jury(c.agreement)]
         ordered = _interleave_by_sample(splits) + _interleave_by_sample(unanimous)

src/ocr_bench/web.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import html
 import io
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
@@ -67,8 +68,10 @@ def _build_pair_summary_html(comparisons: list[dict[str, Any]]) -> str:
 
     rows = []
     for (ma, mb), counts in sorted(pair_counts.items()):
-        short_a = _short_model(ma)
-        short_b = _short_model(mb)
+        # Model names come from dataset columns — escape before embedding in
+        # HTML that the template renders with `| safe`.
+        short_a = html.escape(_short_model(ma))
+        short_b = html.escape(_short_model(mb))
         wins, losses, ties = counts["W"], counts["L"], counts["T"]
         rows.append(
             f"<tr><td>{short_a}</td><td>{short_b}</td>"
@@ -151,7 +154,10 @@ def create_app(
         img_loader = ImageLoader(source_dataset, from_prs=from_prs)
 
     validation_comps = build_validation_comparisons(
-        comparison_rows, n=n_validate, prioritize_splits=True
+        comparison_rows,
+        leaderboard_rows=leaderboard_rows,
+        n=n_validate,
+        prioritize_splits=True,
     )
 
     models = sorted(
@@ -303,12 +309,15 @@ def create_app(
 
     @app.get("/", response_class=RedirectResponse)
     async def index():
-        return RedirectResponse(url="/comparisons", status_code=302)
+        return RedirectResponse(url="/leaderboard", status_code=302)
 
     @app.get("/leaderboard", response_class=HTMLResponse)
     async def leaderboard(request: Request):
+        from ocr_bench.publish import _get_model_sizes
+
         # Build human ELO if we have annotations
         human_board = compute_human_elo(state.annotations, state.validation_comps)
+        sizes = _get_model_sizes()
 
         rows = []
         for row in sorted(state.leaderboard_rows, key=lambda r: r.get("elo", 0), reverse=True):
@@ -324,6 +333,7 @@ def create_app(
             rows.append({
                 "model": model,
                 "model_short": short,
+                "params": row.get("params") or sizes.get(model, ""),
                 "elo": round(row.get("elo", 0)),
                 "elo_low": row.get("elo_low"),
                 "elo_high": row.get("elo_high"),
@@ -336,12 +346,32 @@ def create_app(
             })
 
         has_ci = any(r.get("elo_low") is not None for r in rows)
+
+        # Build chart data — params (numeric) vs win%
+        chart_points = []
+        for r in rows:
+            params_str = r.get("params", "")
+            if params_str:
+                try:
+                    params_num = float(params_str.rstrip("B"))
+                except ValueError:
+                    continue
+                chart_points.append({
+                    "name": r["model_short"],
+                    "params": params_num,
+                    "win_pct": r["win_pct"],
+                    "elo": r["elo"],
+                    "elo_low": r.get("elo_low"),
+                    "elo_high": r.get("elo_high"),
+                })
+
         return templates.TemplateResponse(request, "leaderboard.html", {
             "active_tab": "leaderboard",
             "repo_id": state.repo_id,
             "rows": rows,
             "has_ci": has_ci,
             "has_human_elo": human_board is not None,
+            "chart_points": chart_points,
         })
 
     @app.get("/comparisons", response_class=HTMLResponse)
@@ -441,7 +471,11 @@ def create_app(
         ctx["next_url"] = (
             f"/comparisons/{next_nav}"
             + (f"?winner={winner_filter}" if winner_filter != "All" else "")
-            + (f"{'&' if winner_filter != 'All' else '?'}model={model_filter}" if model_filter != "All" else "")
+            + (
+                f"{'&' if winner_filter != 'All' else '?'}model={model_filter}"
+                if model_filter != "All"
+                else ""
+            )
             if next_nav is not None
             else None
         )