qwen_test_runner/vision/report.py

"""
report.py — Leaderboards + the no-finetune labeler verdict.

Aggregates per-(model, reasoning, category, mode) metrics into:
  * a QUALITY leaderboard ranked by labeler_score (native json_mode columns),
  * a FLEET leaderboard folding throughput, and
  * a verdict naming the best NO-FINETUNE model — or, if none is natively
    sufficient, the best finetune-candidate to feed the SFT/LoRA pipeline.

Native columns use json_mode (no grammar crutch) because that is the signal for
whether a model emits robust JSON on its own.
"""

from __future__ import annotations

import json
from pathlib import Path
from typing import Optional

from .metrics import labeler_score
from .throughput import fleet_score

# Bucketing thresholds (config-surfaceable later).
ACC_FLOOR = 0.55           # below this the vision itself is too weak
NATIVE_ROBUST = 0.90       # native json_mode robustness to count as "ships as-is"


def _bucket(accuracy: Optional[float], native_robust: float) -> str:
    if accuracy is None:
        return "no_task_gt"
    if accuracy < ACC_FLOOR:
        return "insufficient"
    if native_robust >= NATIVE_ROBUST:
        return "native_capable"
    return "finetune_candidate"


def _mean(xs):
    xs = [x for x in xs if x is not None]
    return sum(xs) / len(xs) if xs else None


def summarize(metric_rows: list[dict]) -> list[dict]:
    """Collapse per-category metric rows into one summary per (model, reasoning).

    Native (json_mode) columns drive the no-finetune decision; the constrained
    rows are kept only to compute the native-vs-constrained validity gap.
    """
    by_model: dict[tuple[str, str], dict] = {}
    for r in metric_rows:
        key = (r["model"], r["reasoning"])
        by_model.setdefault(key, {"json_mode": [], "constrained": []})
        bucket = r["mode"] if r["mode"] in ("json_mode", "constrained") else "json_mode"
        by_model[key][bucket].append(r)

    summaries = []
    for (model, reasoning), modes in by_model.items():
        jm = modes["json_mode"] or modes["constrained"]
        acc = _mean([r["primary_score_mean"] for r in jm if r["has_task_score"]])
        valid = _mean([r["schema_valid_rate"] for r in jm]) or 0.0
        robust = _mean([r["json_robustness"] for r in jm]) or 0.0
        constrained_valid = _mean([r["schema_valid_rate"] for r in modes["constrained"]])
        gap = (constrained_valid - valid) if constrained_valid is not None else None
        tok_s = _mean([r["tokens_per_sec"] for r in jm]) or 0.0
        mean_out = _mean([r["mean_output_tokens"] for r in jm]) or 0.0
        lab = labeler_score(acc, valid, robust)
        # samples/hour from tok/s + mean output tokens (prefill folded in elsewhere)
        sph = (3600.0 * tok_s / mean_out) if (tok_s > 0 and mean_out > 0) else 0.0
        summaries.append({
            "model": model, "reasoning": reasoning,
            "accuracy": acc, "native_valid": valid, "native_robust": robust,
            "constrained_valid": constrained_valid, "native_gap": gap,
            "labeler_score": lab, "tokens_per_sec": tok_s, "samples_per_hour": sph,
            "fleet_score": fleet_score(lab, sph) if lab is not None else None,
            "bucket": _bucket(acc, robust),
        })
    summaries.sort(key=lambda s: (s["labeler_score"] is not None, s["labeler_score"] or -1), reverse=True)
    return summaries


def _fmt(x, pct=False):
    if x is None:
        return "n/a"
    return f"{x:.1%}" if pct else f"{x:.3f}"


def quality_table(summaries: list[dict]) -> str:
    lines = [
        "| rank | model | reason | acc | native_valid | native_robust | gap | labeler | tok/s | bucket |",
        "|------|-------|--------|-----|--------------|---------------|-----|---------|-------|--------|",
    ]
    for i, s in enumerate(summaries, 1):
        lines.append(
            f"| {i} | {s['model']} | {s['reasoning']} | {_fmt(s['accuracy'])} | "
            f"{_fmt(s['native_valid'], True)} | {_fmt(s['native_robust'], True)} | "
            f"{_fmt(s['native_gap'], True)} | {_fmt(s['labeler_score'])} | "
            f"{s['tokens_per_sec']:.0f} | {s['bucket']} |"
        )
    return "\n".join(lines)


def fleet_table(summaries: list[dict]) -> str:
    fs = sorted([s for s in summaries if s["fleet_score"] is not None],
                key=lambda s: s["fleet_score"], reverse=True)
    lines = [
        "| rank | model | reason | labeler | samples/hr | fleet |",
        "|------|-------|--------|---------|------------|-------|",
    ]
    for i, s in enumerate(fs, 1):
        lines.append(
            f"| {i} | {s['model']} | {s['reasoning']} | {_fmt(s['labeler_score'])} | "
            f"{s['samples_per_hour']:.0f} | {_fmt(s['fleet_score'])} |"
        )
    return "\n".join(lines)


def verdict(summaries: list[dict]) -> str:
    native = [s for s in summaries if s["bucket"] == "native_capable"]
    ft = [s for s in summaries if s["bucket"] == "finetune_candidate"]
    out = ["## Headline recommendation", ""]
    if native:
        b = native[0]
        out.append(
            f"> **Best no-finetune labeler:** `{b['model']}` ({b['reasoning']}) — "
            f"{_fmt(b['native_valid'], True)} native-valid, {_fmt(b['accuracy'])} accuracy, "
            f"{b['tokens_per_sec']:.0f} tok/s. Ships as-is."
        )
    elif ft:
        b = ft[0]
        out.append(
            f"> **No natively-sufficient model.** Best **finetune-candidate:** `{b['model']}` "
            f"({b['reasoning']}) — robust vision ({_fmt(b['accuracy'])} acc) but native JSON gap = "
            f"{_fmt(b['native_gap'], True)}. Close it with the existing data-gen → SFT/LoRA pipeline."
        )
    else:
        out.append("> No model cleared the accuracy floor on this run. Re-check inputs / categories.")
    fleet = sorted([s for s in summaries if s["fleet_score"] is not None],
                   key=lambda s: s["fleet_score"], reverse=True)
    if fleet:
        f = fleet[0]
        out.append("")
        out.append(
            f"> **Best fleet labeler (1M+ images):** `{f['model']}` ({f['reasoning']}) — "
            f"{f['samples_per_hour']:.0f} samples/hr at labeler {_fmt(f['labeler_score'])}."
        )
    return "\n".join(out)


def write_reports(run_dir: Path, metric_rows: list[dict], config: dict) -> dict:
    summaries = summarize(metric_rows)
    md = [
        f"# Qwen VLM Labeler Selection — {run_dir.name}",
        "",
        f"models={config.get('models')}  categories={len(config.get('categories', []))}  "
        f"reasoning={config.get('reasonings')}  modes={config.get('modes')}  "
        f"n={config.get('n')}  dataset={config.get('dataset')}  runner={config.get('runner')}",
        "",
        "## Quality leaderboard (labeler_score, native json_mode)",
        "",
        quality_table(summaries),
        "",
        "## Fleet leaderboard (accuracy × throughput)",
        "",
        fleet_table(summaries),
        "",
        verdict(summaries),
        "",
    ]
    (run_dir / "leaderboard.md").write_text("\n".join(md), encoding="utf-8")
    (run_dir / "summary.json").write_text(
        json.dumps({"config": config, "summaries": summaries}, indent=2), encoding="utf-8")
    # CSV
    cols = ["model", "reasoning", "accuracy", "native_valid", "native_robust", "native_gap",
            "labeler_score", "tokens_per_sec", "samples_per_hour", "fleet_score", "bucket"]
    csv_lines = [",".join(cols)]
    for s in summaries:
        csv_lines.append(",".join("" if s.get(c) is None else str(s.get(c)) for c in cols))
    (run_dir / "leaderboard.csv").write_text("\n".join(csv_lines), encoding="utf-8")
    return {"summaries": summaries}