server/scripts/build_benchmark_from_appendices.py

"""Build server/data/benchmark.json from AIChemist-Lab appendix source tables."""

from __future__ import annotations

import csv
import json
from pathlib import Path
from statistics import mean

ROOT = Path(__file__).resolve().parents[2]
SOURCES = ROOT / "docs" / "appendices" / "sources"
OUT = ROOT / "server" / "data" / "benchmark.json"

MODELS = [
    {
        "name": "GPT-4o-mini",
        "type": "Standard",
        "provider": "OpenAI",
        "access": "API",
        "costPer1mTokens": "$0.15",
        "latency": "2.1s",
        "entity_cols": ["GPT-4o-mini"],
        "route_cols": ["GPT-4o-mini"],
        "eval_key": "gpt-4o-mini",
        "ddi_cols": ["GPT-4o-mini"],
        "ddi_paper_cols": ["GPT-4o-mini"],
        "rx_llm_cols": ["GPT-4o-mini"],
    },
    {
        "name": "GPT-5 Chat",
        "type": "Reasoning",
        "provider": "OpenAI",
        "access": "API",
        "costPer1mTokens": "$1.10",
        "latency": "8.2s",
        "entity_cols": ["GPT-5 Chat"],
        "route_cols": ["GPT-5-chat"],
        "eval_key": "azure-gpt-5-chat",
        "ddi_cols": ["GPT-5-Chat"],
        "ddi_paper_cols": ["GPT-5-Chat"],
        "rx_llm_cols": ["GPT-5-Chat"],
    },
    {
        "name": "MedGemma-27B",
        "type": "Medical",
        "provider": "Google",
        "access": "Open Weights",
        "costPer1mTokens": "$0.15",
        "latency": "1.8s",
        "entity_cols": [],
        "route_cols": [],
        "eval_key": "",
        "ddi_cols": [],
        "ddi_paper_cols": ["MedGemma-27B"],
        "rx_llm_cols": ["MedGemma-27B"],
    },
    {
        "name": "Gemma 3 27B",
        "type": "Standard",
        "provider": "Google",
        "access": "Open Weights",
        "costPer1mTokens": "$0.15",
        "latency": "1.8s",
        "entity_cols": ["Gemma3"],
        "route_cols": ["Gemma-3-27B-IT"],
        "eval_key": "google_gemma-3-27b-it",
        "ddi_cols": ["Gemma-27B"],
        "ddi_paper_cols": [],
        "rx_llm_cols": [],
    },
    {
        "name": "Llama 3.3 70B",
        "type": "Standard",
        "provider": "Meta",
        "access": "Open Weights",
        "costPer1mTokens": "$0.70",
        "latency": "3.2s",
        "entity_cols": ["LLaMA3"],
        "route_cols": ["LLaMA-3.3-70B-Instruct"],
        "eval_key": "meta-llama_Llama-3.3-70B-Instruct",
        "ddi_cols": ["LLaMA3-70B"],
        "ddi_paper_cols": ["LLaMA3-70B"],
        "rx_llm_cols": ["LLaMA3-70B"],
    },
    {
        "name": "Qwen3 32B",
        "type": "Standard",
        "provider": "Alibaba",
        "access": "Open Weights",
        "costPer1mTokens": "$0.20",
        "latency": "2.5s",
        "entity_cols": ["Qwen3"],
        "route_cols": ["Qwen3-32B"],
        "eval_key": "Qwen_Qwen3-32B",
        "ddi_cols": ["Qwen3-32B"],
        "ddi_paper_cols": ["Qwen3-32B"],
        "rx_llm_cols": ["Qwen3-32B"],
    },
    {
        "name": "DrugGPT",
        "type": "Standard",
        "provider": "DrugGPT",
        "access": "Specialized",
        "costPer1mTokens": "N/A",
        "latency": "N/A",
        "entity_cols": [],
        "route_cols": [],
        "eval_key": "druggpt",
        "ddi_cols": [],
        "ddi_paper_cols": ["DrugGPT"],
        "rx_llm_cols": ["DrugGPT"],
    },
]

# Pokemon appendix (PMC Table 2, default drug-dosing confabulation %). Suspicion detected = 100 - rate.
POKEMON_CONFAB_DEFAULT_DOSING = {
    "GPT-4o-mini": {"generic": 97.7, "brand": 98.8},
    "Llama 3.3 70B": {"generic": 86.0, "brand": 91.9},
    "Gemma 3 27B": {"generic": 95.9, "brand": 97.7},
    "Qwen3 32B": {"generic": 98.4, "brand": 98.8},
}

MEDMATCH_CATEGORY_MAP = {
    "Oral solid (n=40)": "MedMatch (Oral Solid)",
    "Oral liquid (n=10)": "MedMatch (Oral Liq)",
    "Intravenous intermittent (n=17)": "MedMatch (IV Intermit)",
    "Intravenous push (n=17)": "MedMatch (IV Push)",
    "Intravenous continuous infusion titratable (n=11)": "MedMatch (Continuous Titrate)",
    "Intravenous continuous infusion non-titratable (n=6)": "MedMatch (Continuous Non-Titrate)",
}

MEDMATCH_PAPER_NOTE = (
    "MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; "
    "one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots."
)
MEDMATCH_FORMAT_NOTE = (
    "Convert free-text medication order to standardized MedMatch JSON slot format per administration class."
)

RX_BENCH_NOTE = "Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials."
DDI_PAPER_NOTE = "DDI identification paper: 750 clinician-annotated DDI scenarios. Zero-shot; precision, recall, F1, accuracy, self-consistency."
POKEMON_JUDGE = (
    "Suspects fictitious | Inherited confabulation (answered as if real drug) | "
    "Epistemic confabulation (replaced fictitious drug with real medication)"
)

TASK_DEFINITIONS = [
    {
        "name": "Formulation Matching",
        "prompt": f"Generic drug name (e.g., amlodipine). List all FDA-approved dosage forms. {RX_BENCH_NOTE}",
        "response": "Complete and correct list of formulations.",
        "humanAnnotation": "Complete and correct list of formulations.",
        "agreement": "96%",
        "metrics": ["Precision", "Recall", "F1-score", "Accuracy", "Correctness consistency"],
    },
    {
        "name": "Drug Order Gen (Sig)",
        "prompt": f"Generic drug name (e.g., carvedilol). Generate one clinically appropriate complete oral medication order (sig). {RX_BENCH_NOTE}",
        "response": "One clinically appropriate complete medication order.",
        "humanAnnotation": "Clinically appropriate complete medication order.",
        "agreement": "98%",
        "metrics": ["Exact match", "HAMeC score", "Correctness consistency"],
    },
    {
        "name": "Route Matching",
        "prompt": f"Generic drug name (e.g., prednisolone). List all safe routes of administration. {RX_BENCH_NOTE}",
        "response": "Complete and correct list of routes.",
        "humanAnnotation": "Complete and correct list of routes.",
        "agreement": "95%",
        "metrics": ["Precision", "Recall", "F1-score", "Accuracy", "Correctness consistency"],
    },
    {
        "name": "Rx-Bench DDI ID",
        "prompt": (
            "Pointwise two-drug classification: identify clinically significant interacting pair "
            f"(Category C, D, or X) from a medication list with full dosing. {RX_BENCH_NOTE}"
        ),
        "response": "Correct interacting drug pair.",
        "humanAnnotation": "Correct interacting drug pair.",
        "agreement": "94%",
        "metrics": ["Precision", "Recall", "F1-score", "Accuracy", "Self-consistency"],
    },
    {
        "name": "Renal Dose ID",
        "prompt": f"Generic drug name (e.g., vancomycin). Determine if renal dose adjustment is required (Yes/No). {RX_BENCH_NOTE}",
        "response": "Yes or No.",
        "humanAnnotation": "Yes or No.",
        "agreement": "99%",
        "metrics": ["Precision", "Recall", "F1-score", "Exact match", "Correctness consistency"],
    },
    {
        "name": "Drug-Indication",
        "prompt": f"Drug name. Identify FDA-approved clinical indications. {RX_BENCH_NOTE}",
        "response": "Correct list of FDA-approved indications.",
        "humanAnnotation": "Correct list of FDA-approved indications.",
        "agreement": "93%",
        "metrics": ["Precision", "Recall", "F1-score", "Accuracy", "Correctness consistency"],
    },
    {
        "name": "DDI ID",
        "prompt": (
            "Pointwise DDI identification: classify clinically significant two-drug interactions "
            f"from the DDI identification paper. {DDI_PAPER_NOTE}"
        ),
        "response": "Correct pointwise DDI classification.",
        "humanAnnotation": "Correct pointwise DDI classification.",
        "agreement": "94%",
        "metrics": ["Precision", "Recall", "F1-score", "Accuracy", "Self-consistency"],
    },
    {
        "name": "DDI Verification",
        "prompt": (
            "Drug pair with proposed interaction category and clinical action. "
            "Verify whether the proposed interaction assessment is correct (hedging/default prompt). "
            "Source: LLM-Uncertainty-DDI supplement."
        ),
        "response": '"A" (Correct) or "B" (Incorrect).',
        "humanAnnotation": '"A" (Correct) or "B" (Incorrect).',
        "agreement": "97%",
        "metrics": ["Correct answer rate", "Refusal rate", "Correct given attempted"],
    },
    {
        "name": "DDI 3-Drug Combo",
        "prompt": f"Pairwise three-drug discrimination: identify interacting pair(s) from three medications with full dosing. {DDI_PAPER_NOTE}",
        "response": "Correct interacting pair(s).",
        "humanAnnotation": "Correct interacting pair(s).",
        "agreement": "90%",
        "metrics": ["Precision", "Recall", "F1-score", "Accuracy", "Self-consistency"],
    },
    {
        "name": "DDI Multi-Drug",
        "prompt": f"Listwise 4–6 drug selection: identify all interacting pairs from a polypharmacy regimen. {DDI_PAPER_NOTE}",
        "response": "All correct interacting drug pairs.",
        "humanAnnotation": "All correct interacting drug pairs.",
        "agreement": "82%",
        "metrics": ["Precision", "Recall", "F1-score", "Accuracy", "Self-consistency"],
    },
    {
        "name": "MedMatch (Oral Solid)",
        "prompt": f"{MEDMATCH_FORMAT_NOTE} Oral solid (n=40). {MEDMATCH_PAPER_NOTE}",
        "response": "[drug name][numerical dose][abbreviated unit strength of dose][amount][formulation] by mouth [frequency]",
        "humanAnnotation": "Exact JSON slot match for oral solid MedMatch format.",
        "agreement": "91%",
        "metrics": ["MedMatch score (exact field match)", "Micro-F1"],
    },
    {
        "name": "MedMatch (Oral Liq)",
        "prompt": f"{MEDMATCH_FORMAT_NOTE} Oral liquid (n=10). {MEDMATCH_PAPER_NOTE}",
        "response": "[drug name][numerical dose][abbreviated unit strength of dose][numerical volume][abbreviated unit strength of volume] of the [concentration][formulation unit] [formulation] by mouth [frequency]",
        "humanAnnotation": "Exact JSON slot match for oral liquid MedMatch format.",
        "agreement": "90%",
        "metrics": ["MedMatch score (exact field match)", "Micro-F1"],
    },
    {
        "name": "MedMatch (IV Intermit)",
        "prompt": f"{MEDMATCH_FORMAT_NOTE} IV intermittent (n=17). {MEDMATCH_PAPER_NOTE}",
        "response": "[drug name][numerical dose][abbreviated unit strength of dose][amount of diluent volume][volume unit][compatible diluent type] intravenously infused over [infusion time] [frequency]",
        "humanAnnotation": "Exact JSON slot match for IV intermittent MedMatch format.",
        "agreement": "88%",
        "metrics": ["MedMatch score (exact field match)", "Micro-F1"],
    },
    {
        "name": "MedMatch (IV Push)",
        "prompt": f"{MEDMATCH_FORMAT_NOTE} IV push (n=17). {MEDMATCH_PAPER_NOTE}",
        "response": "[drug name][numerical dose][abbreviated unit strength of dose][amount of volume][volume unit] of the [concentration][concentration unit][formulation] intravenous push [frequency]",
        "humanAnnotation": "Exact JSON slot match for IV push MedMatch format.",
        "agreement": "89%",
        "metrics": ["MedMatch score (exact field match)", "Micro-F1"],
    },
    {
        "name": "MedMatch (Continuous Titrate)",
        "prompt": f"{MEDMATCH_FORMAT_NOTE} IV continuous titratable (n=11). {MEDMATCH_PAPER_NOTE}",
        "response": '[drug name][numerical dose][abbreviated unit strength of dose] "in" [diluent volume][volume unit][compatible diluent type] "continuous intravenous infusion starting at" [starting rate][unit of measure] "titrated by" [titration dose][titration unit] [titration frequency] to achieve [titration goal]',
        "humanAnnotation": "Exact JSON slot match for continuous titratable infusion.",
        "agreement": "85%",
        "metrics": ["MedMatch score (exact field match)", "Micro-F1"],
    },
    {
        "name": "MedMatch (Continuous Non-Titrate)",
        "prompt": f"{MEDMATCH_FORMAT_NOTE} IV continuous non-titratable (n=6). {MEDMATCH_PAPER_NOTE}",
        "response": '[drug name][numerical dose][abbreviated unit strength of dose][diluent volume][volume unit]"in"[compatible diluent type] "continuous intravenous infusion at" [rate][unit of measure]',
        "humanAnnotation": "Exact JSON slot match for continuous non-titratable infusion.",
        "agreement": "87%",
        "metrics": ["MedMatch score (exact field match)", "Micro-F1"],
    },
    {
        "name": "MedMatch Route Selection",
        "prompt": (
            "Route omitted from medication prompt. Classify order into administration route category: "
            "oral solid, oral liquid, IV intermittent, IV push, or IV continuous (titratable/non-titratable). "
            f"{MEDMATCH_PAPER_NOTE} Dataset posted at github.com/AIChemist-Lab/MedMatch."
        ),
        "response": "Correct route category (by mouth, IV push, IV intermittent, or IV continuous).",
        "humanAnnotation": "Correct route category assignment.",
        "agreement": "88%",
        "metrics": ["Route accuracy", "MedMatch score"],
    },
    {
        "name": "Pokémon (Generic)",
        "prompt": (
            "250 medication vignettes: 4–6 real generic medications plus one fabricated Pokémon "
            "medication with complete dosing (drug, dose, unit, route, frequency). "
            "Task: provide dosing range or indication. Drug or Pokémon? paper (PMC12870567)."
        ),
        "response": POKEMON_JUDGE,
        "humanAnnotation": POKEMON_JUDGE,
        "agreement": "86%",
        "metrics": ["LLM-as-a-judge", "Confabulation rate"],
    },
    {
        "name": "Pokémon (Brand)",
        "prompt": (
            "250 medication vignettes: 4–6 real brand medications plus one fabricated Pokémon "
            "medication with complete dosing. Task: provide dosing range or indication. "
            "Drug or Pokémon? paper (PMC12870567)."
        ),
        "response": POKEMON_JUDGE,
        "humanAnnotation": POKEMON_JUDGE,
        "agreement": "85%",
        "metrics": ["LLM-as-a-judge", "Confabulation rate"],
    },
]


def parse_pct(value: str) -> float:
    return float(value.strip().replace("%", ""))


def load_entity_table() -> dict[str, dict[str, float]]:
    """Returns MedMatch category field-average scores by model."""
    path = SOURCES / "entity_accuracy_table.csv"
    category_avgs: dict[str, dict[str, list[float]]] = {}
    cur = None
    with path.open(newline="", encoding="utf-8") as f:
        reader = csv.reader(f)
        header = next(reader)
        col_index = {name: i for i, name in enumerate(header)}
        for row in reader:
            if not row or not row[0]:
                continue
            if row[0].endswith(")"):
                cur = row[0]
                category_avgs[cur] = {m["name"]: [] for m in MODELS}
                continue
            if cur is None:
                continue
            entity = row[0].strip()
            for model in MODELS:
                for col in model["entity_cols"]:
                    idx = col_index.get(col)
                    if idx is None or not row[idx].strip():
                        continue
                    val = parse_pct(row[idx])
                    category_avgs[cur][model["name"]].append(val)
    category_means = {
        cat: {m: round(mean(vals), 1) if vals else 0.0 for m, vals in per_model.items()}
        for cat, per_model in category_avgs.items()
    }
    return category_means


def load_route_table() -> dict[str, float]:
    path = SOURCES / "route_accuracy_table.csv"
    per_model: dict[str, list[float]] = {m["name"]: [] for m in MODELS}
    with path.open(newline="", encoding="utf-8") as f:
        reader = csv.reader(f)
        header = next(reader)
        col_index = {name: i for i, name in enumerate(header)}
        for row in reader:
            if not row or row[0] == "Medication Route":
                continue
            for model in MODELS:
                for col in model["route_cols"]:
                    idx = col_index.get(col)
                    if idx is None or not row[idx].strip():
                        continue
                    per_model[model["name"]].append(parse_pct(row[idx]))
    return {m: round(mean(vals), 1) if vals else 0.0 for m, vals in per_model.items()}


def load_ddi_identification_table3() -> dict[str, dict[str, float]]:
    """DDI identification paper Table 3 experiment-level accuracy rows."""
    path = SOURCES / "ddi_identification_table3.csv"
    out: dict[str, dict[str, float]] = {m["name"]: {} for m in MODELS}
    with path.open(newline="", encoding="utf-8") as f:
        reader = csv.reader(f)
        header = next(reader)
        col_index = {name: i for i, name in enumerate(header)}
        for row in reader:
            if not row:
                continue
            task = row[0].strip()
            for model in MODELS:
                for col in model["ddi_paper_cols"]:
                    idx = col_index.get(col)
                    if idx is None or not row[idx].strip():
                        continue
                    out[model["name"]][task] = parse_pct(row[idx])
    return out


def load_rx_llm_primary_metrics() -> dict[str, dict[str, float]]:
    """Rx-Bench Tables 2-3 primary task metrics used in Figure 2."""
    path = SOURCES / "rx_llm_tables_2_3.csv"
    out: dict[str, dict[str, float]] = {m["name"]: {} for m in MODELS}
    with path.open(newline="", encoding="utf-8") as f:
        reader = csv.reader(f)
        header = next(reader)
        col_index = {name: i for i, name in enumerate(header)}
        for row in reader:
            if not row:
                continue
            source_task = row[0].strip()
            task = "Rx-Bench DDI ID" if source_task == "DDI ID" else source_task
            for model in MODELS:
                for col in model["rx_llm_cols"]:
                    idx = col_index.get(col)
                    if idx is None or not row[idx].strip():
                        continue
                    out[model["name"]][task] = parse_pct(row[idx])
    return out


def load_ddi_verification_accuracy() -> dict[str, float]:
    """LLM-Uncertainty-DDI supplement: verification task default-prompt accuracy."""
    path = SOURCES / "table_4_results.csv"
    out: dict[str, float] = {}
    with path.open(newline="", encoding="utf-8") as f:
        reader = csv.reader(f)
        header = next(reader)
        col_index = {name: i for i, name in enumerate(header)}
        for row in reader:
            if len(row) < 3:
                continue
            prompt = row[0].strip().strip('"')
            metric = row[1].strip().strip('"')
            if prompt == "Default Prompt with Hedging" and metric == "Correct Answer Rate":
                for model in MODELS:
                    for col in model["ddi_cols"]:
                        idx = col_index.get(col)
                        if idx is None:
                            continue
                        raw = row[idx].strip().strip('"')
                        out[model["name"]] = parse_pct(raw.split("[")[0].strip())
                break
    return out


def earned_failed(score: float) -> dict[str, float]:
    earned = max(0, min(100, round(score, 1)))
    return {"earned": earned, "failed": round(100 - earned, 1)}


def build() -> dict:
    category_means = load_entity_table()
    route_selection_means = load_route_table()
    rx_llm_primary_metrics = load_rx_llm_primary_metrics()
    ddi_paper_acc = load_ddi_identification_table3()
    ddi_verification_acc = load_ddi_verification_accuracy()

    task_scores: dict[str, dict[str, float]] = {m["name"]: {} for m in MODELS}

    for model in MODELS:
        name = model["name"]
        for rx_task, score in rx_llm_primary_metrics.get(name, {}).items():
            task_scores[name][rx_task] = score
        if model["route_cols"]:
            task_scores[name]["MedMatch Route Selection"] = route_selection_means[name]
        for ddi_task in ["DDI ID", "DDI 3-Drug Combo", "DDI Multi-Drug"]:
            if ddi_task in ddi_paper_acc.get(name, {}):
                task_scores[name][ddi_task] = ddi_paper_acc[name][ddi_task]
        if name in ddi_verification_acc:
            task_scores[name]["DDI Verification"] = ddi_verification_acc[name]
        if model["entity_cols"]:
            for cat, task in MEDMATCH_CATEGORY_MAP.items():
                task_scores[name][task] = category_means[cat][name]
        pokemon = POKEMON_CONFAB_DEFAULT_DOSING.get(name)
        if pokemon:
            task_scores[name]["Pokémon (Generic)"] = round(100 - pokemon["generic"], 1)
            task_scores[name]["Pokémon (Brand)"] = round(100 - pokemon["brand"], 1)

    def avg_tasks(name: str, tasks: list[str]) -> float | None:
        vals = [task_scores[name][t] for t in tasks if t in task_scores[name]]
        return round(mean(vals) / 100, 3) if vals else None

    def avg_rx_llm_tasks(name: str, tasks: list[str]) -> float | None:
        vals = [rx_llm_primary_metrics.get(name, {}).get(t) for t in tasks]
        vals = [v for v in vals if v is not None]
        return round(mean(vals) / 100, 3) if vals else None

    def score_value(score: float | None) -> str:
        return f"{score:.3f}" if score is not None else "N/A"

    ddi_paper_tasks = ["DDI ID", "DDI 3-Drug Combo", "DDI Multi-Drug"]
    rx_llm_tasks = [
        "Formulation Matching",
        "Drug Order Gen (Sig)",
        "Route Matching",
        "Rx-Bench DDI ID",
        "Renal Dose ID",
        "Drug-Indication",
    ]
    medmatch_tasks = list(MEDMATCH_CATEGORY_MAP.values()) + ["MedMatch Route Selection"]
    pokemon_tasks = ["Pokémon (Generic)", "Pokémon (Brand)"]

    models_out = []
    for model in MODELS:
        name = model["name"]
        rx_llm_score = avg_rx_llm_tasks(name, rx_llm_tasks)
        ddi_score = avg_tasks(name, ddi_paper_tasks)
        medmatch_score = avg_tasks(name, medmatch_tasks)
        pokemon_score = avg_tasks(name, pokemon_tasks)
        reported_scores = [s for s in [rx_llm_score, ddi_score, medmatch_score, pokemon_score] if s is not None]
        win_rate = round(mean(reported_scores), 3) if reported_scores else 0.0
        models_out.append({
            "name": name,
            "type": model["type"],
            "provider": model["provider"],
            "access": model["access"],
            "winRate": win_rate,
            "costPer1mTokens": model["costPer1mTokens"],
            "latency": model["latency"],
            "isCustom": False,
        })

    benchmark_results = []
    for model in MODELS:
        name = model["name"]
        for task_name, score in task_scores[name].items():
            row = earned_failed(score)
            benchmark_results.append({
                "modelName": name,
                "taskName": task_name,
                "earned": row["earned"],
                "failed": row["failed"],
            })

    leaderboard_scores = []
    for model in MODELS:
        name = model["name"]
        rx_llm_score = avg_rx_llm_tasks(name, rx_llm_tasks)
        ddi_score = avg_tasks(name, ddi_paper_tasks)
        medmatch_score = avg_tasks(name, medmatch_tasks)
        pokemon_score = avg_tasks(name, pokemon_tasks)
        reported_scores = [s for s in [rx_llm_score, ddi_score, medmatch_score, pokemon_score] if s is not None]
        macro_win = round(mean(reported_scores), 3) if reported_scores else None
        source_coverage = f"{len(reported_scores)}/4"

        accuracy_rows = {
            "Mean Win Rate": score_value(macro_win),
            "Rx-Bench (CMM)": score_value(rx_llm_score),
            "DDI Identification": score_value(ddi_score),
            "MedMatch": score_value(medmatch_score),
            "Drug or Pokémon?": score_value(pokemon_score),
        }
        efficiency_rows = {
            "Cost (per 1M tokens)": model["costPer1mTokens"],
            "Latency (s / request)": model["latency"],
        }
        general_rows = {
            "Provider": model["provider"],
            "Access": model["access"],
            "Model Type": model["type"],
            "Source Coverage": source_coverage,
        }
        for metric, value in accuracy_rows.items():
            leaderboard_scores.append({"modelName": name, "metricName": metric, "tab": "Accuracy", "value": value})
        for metric, value in efficiency_rows.items():
            leaderboard_scores.append({"modelName": name, "metricName": metric, "tab": "Efficiency", "value": value})
        for metric, value in general_rows.items():
            leaderboard_scores.append({"modelName": name, "metricName": metric, "tab": "General information", "value": value})

    return {
        "models": models_out,
        "benchmarkResults": benchmark_results,
        "leaderboardScores": leaderboard_scores,
        "taskDefinitions": TASK_DEFINITIONS,
        "meta": {
            "papers": {
                "rxLlm": {
                    "title": "Rx-Bench",
                    "doi": "10.64898/2025.12.01.25341004",
                    "url": "https://www.medrxiv.org/content/10.64898/2025.12.01.25341004v2",
                },
                "ddiIdentification": {
                    "title": "Drug-drug interaction identification using large language models",
                    "doi": "10.64898/2025.12.03.25341549",
                    "url": "https://www.medrxiv.org/content/10.64898/2025.12.03.25341549v2",
                },
                "pokemon": {
                    "title": "Drug or Pokémon?",
                    "pmc": "PMC12870567",
                    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12870567/",
                },
                "medMatch": {
                    "title": "MedMatch: a first step for the automation of large language model performance benchmarking for medication-related tasks",
                    "doi": "10.64898/2026.01.13.26343949",
                    "pmc": "PMC12870651",
                    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12870651/",
                    "github": "https://github.com/AIChemist-Lab/MedMatch",
                },
            },
            "sources": [
                "Rx-Bench submission 4.8.26 / medRxiv 10.64898/2025.12.01.25341004",
                "DDI identification medRxiv 10.64898/2025.12.03.25341549",
                "MedMatch medRxiv 10.64898/2026.01.13.26343949 / PMC12870651",
                "Drug or Pokémon? PMC12870567",
                "github.com/AIChemist-Lab/MedMatch, LLM-Uncertainty-DDI appendix tables",
            ],
            "supportedModels": [m["name"] for m in MODELS],
            "scorePolicy": {
                "reportedMean": "Mean Win Rate averages only source-backed paper scores and excludes N/A cells.",
                "sourceCoverage": "Number of primary papers with source-backed performance for the model out of four.",
                "rxLlm": "Rx-Bench (CMM) is the macro mean of the six primary task metrics reported in Rx-Bench Tables 2-3; task-level scores remain available in Scenarios.",
                "pokemon": "Drug or Pokémon? scores are suspicion detected = 100 - default-dosing confabulation rate; unreported models are N/A, not zero.",
            },
            "note": "Mean Win Rate is the reported mean over source-backed paper scores only. Rx-Bench (CMM) is the macro mean of six primary task metrics from Rx-Bench Tables 2-3; task-level scores remain attached to Scenarios rather than the main leaderboard. MedGemma-27B is listed separately where source tables report MedGemma rather than base Gemma 3 27B. DrugGPT scores now include Rx-Bench and DDI identification source tables, but remain N/A for MedMatch and Drug or Pokémon? where not reported. DDI Verification remains a supplemental LLM-Uncertainty-DDI row and is not part of the four-paper reported mean. GPT-5 Chat and DrugGPT were not evaluated in the Drug or Pokémon? source table. Cost and Latency are indicative estimates and are not source-backed.",
        },
    }


def main() -> None:
    data = build()
    OUT.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
    print(f"Wrote {OUT}")
    print(f"Models: {len(data['models'])}, benchmark rows: {len(data['benchmarkResults'])}")


if __name__ == "__main__":
    main()