[
  {
    "paper_id": "rx-llm",
    "title": "Rx-Bench: a benchmarking suite to evaluate safe large language model performance for medication-related tasks",
    "domain": "Comprehensive Medication Management",
    "doi": "10.64898/2025.12.01.25341004",
    "pmc": null,
    "url": "https://www.medrxiv.org/content/10.64898/2025.12.01.25341004v2",
    "github": "https://github.com/AIChemist-Lab",
    "dashboard_metric": "Rx-Bench (CMM)",
    "dataset_scope": "6 CMM benchmark definitions; 250 clinician-annotated cases per benchmark",
    "source_files": [
      "docs/appendices/sources/rx_llm_tables_2_3.csv",
      "Rx-Bench manuscript Tables 2-3"
    ],
    "notes": "Dashboard score is the macro mean of six primary task metrics from Rx-Bench Tables 2-3; Scenarios exposes each benchmark task separately.",
    "model_scores": [
      {
        "model": "GPT-4o-mini",
        "score_fraction": 0.757,
        "score_percent": 75.7,
        "score_status": "published_table_aggregate"
      },
      {
        "model": "GPT-5 Chat",
        "score_fraction": 0.85,
        "score_percent": 85.0,
        "score_status": "published_table_aggregate"
      },
      {
        "model": "MedGemma-27B",
        "score_fraction": 0.719,
        "score_percent": 71.9,
        "score_status": "published_table_aggregate"
      },
      {
        "model": "Gemma 3 27B",
        "score_fraction": null,
        "score_percent": null,
        "score_status": "not_reported_for_model"
      },
      {
        "model": "Llama 3.3 70B",
        "score_fraction": 0.773,
        "score_percent": 77.3,
        "score_status": "published_table_aggregate"
      },
      {
        "model": "Qwen3 32B",
        "score_fraction": 0.728,
        "score_percent": 72.8,
        "score_status": "published_table_aggregate"
      },
      {
        "model": "DrugGPT",
        "score_fraction": 0.786,
        "score_percent": 78.6,
        "score_status": "published_table_aggregate"
      }
    ],
    "tasks": [
      {
        "task_name": "Formulation Matching",
        "dataset_slice": "250 cases",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Accuracy",
          "Correctness consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/rx_llm_tables_2_3.csv; Rx-Bench manuscript Tables 2-3",
        "notes": "Uses Rx-Bench primary metric (F1) reported in Tables 2-3; base Gemma 3 27B remains N/A because MedGemma-27B is reported separately.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 53.1,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 73.0,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 40.9,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 54.0,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 36.6,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 63.4,
            "score_status": "published_table"
          }
        ]
      },
      {
        "task_name": "Drug Order Gen (Sig)",
        "dataset_slice": "250 cases",
        "metrics": [
          "Exact match",
          "HAMeC score",
          "Correctness consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/rx_llm_tables_2_3.csv; Rx-Bench manuscript Tables 2-3",
        "notes": "Uses Rx-Bench primary metric (Overall accuracy) reported in Tables 2-3; base Gemma 3 27B remains N/A because MedGemma-27B is reported separately.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 83.2,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 90.0,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 81.2,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 88.0,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 79.6,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 80.4,
            "score_status": "published_table"
          }
        ]
      },
      {
        "task_name": "Route Matching",
        "dataset_slice": "250 cases",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Accuracy",
          "Correctness consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/rx_llm_tables_2_3.csv; Rx-Bench manuscript Tables 2-3",
        "notes": "Uses Rx-Bench primary metric (F1) reported in Tables 2-3; base Gemma 3 27B remains N/A because MedGemma-27B is reported separately.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 67.7,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 75.2,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 69.1,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 74.3,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 71.6,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 74.0,
            "score_status": "published_table"
          }
        ]
      },
      {
        "task_name": "Rx-Bench DDI ID",
        "dataset_slice": "pointwise two-drug format",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Accuracy",
          "Self-consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/rx_llm_tables_2_3.csv; Rx-Bench manuscript Tables 2-3",
        "notes": "Uses Rx-Bench primary metric (Accuracy) reported in Tables 2-3; base Gemma 3 27B remains N/A because MedGemma-27B is reported separately.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 70.4,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 84.8,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 66.5,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 66.7,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 75.6,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 77.7,
            "score_status": "published_table"
          }
        ]
      },
      {
        "task_name": "Renal Dose ID",
        "dataset_slice": "250 cases",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Exact match",
          "Correctness consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/rx_llm_tables_2_3.csv; Rx-Bench manuscript Tables 2-3",
        "notes": "Uses Rx-Bench primary metric (F1) reported in Tables 2-3; base Gemma 3 27B remains N/A because MedGemma-27B is reported separately.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 83.3,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 87.4,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 76.9,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 83.2,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 79.7,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 77.9,
            "score_status": "published_table"
          }
        ]
      },
      {
        "task_name": "Drug-Indication",
        "dataset_slice": "250 cases",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Accuracy",
          "Correctness consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/rx_llm_tables_2_3.csv; Rx-Bench manuscript Tables 2-3",
        "notes": "Uses Rx-Bench primary metric (Accuracy) reported in Tables 2-3; base Gemma 3 27B remains N/A because MedGemma-27B is reported separately.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 96.5,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 99.3,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 96.9,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 97.6,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 94.0,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 98.4,
            "score_status": "published_table"
          }
        ]
      }
    ]
  },
  {
    "paper_id": "ddi-identification",
    "title": "Drug-drug interaction identification using large language models",
    "domain": "Drug-drug interaction identification",
    "doi": "10.64898/2025.12.03.25341549",
    "pmc": null,
    "url": "https://www.medrxiv.org/content/10.64898/2025.12.03.25341549v2",
    "github": "https://github.com/AIChemist-Lab/LLM-DDI",
    "dashboard_metric": "DDI Identification",
    "dataset_scope": "750 clinician-annotated DDI scenarios across pointwise, pairwise, and listwise formats",
    "source_files": [
      "docs/appendices/sources/ddi_identification_table3.csv",
      "server/data/benchmark.json"
    ],
    "notes": "Scores are Table 3 experiment-level accuracies.",
    "model_scores": [
      {
        "model": "GPT-4o-mini",
        "score_fraction": 0.73,
        "score_percent": 73,
        "score_status": "published_table"
      },
      {
        "model": "GPT-5 Chat",
        "score_fraction": 0.875,
        "score_percent": 87.5,
        "score_status": "published_table"
      },
      {
        "model": "MedGemma-27B",
        "score_fraction": 0.717,
        "score_percent": 71.7,
        "score_status": "published_table"
      },
      {
        "model": "Gemma 3 27B",
        "score_fraction": null,
        "score_percent": null,
        "score_status": "not_reported_for_model"
      },
      {
        "model": "Llama 3.3 70B",
        "score_fraction": 0.7,
        "score_percent": 70,
        "score_status": "published_table"
      },
      {
        "model": "Qwen3 32B",
        "score_fraction": 0.756,
        "score_percent": 75.6,
        "score_status": "published_table"
      },
      {
        "model": "DrugGPT",
        "score_fraction": 0.788,
        "score_percent": 78.8,
        "score_status": "published_table"
      }
    ],
    "tasks": [
      {
        "task_name": "DDI ID",
        "dataset_slice": "pointwise two-drug format",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Accuracy",
          "Self-consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/ddi_identification_table3.csv; server/data/benchmark.json",
        "notes": "Table 3 row: DDI ID.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 61,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 81,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 50,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 60,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 67,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 50,
            "score_status": "published_table"
          }
        ]
      },
      {
        "task_name": "DDI 3-Drug Combo",
        "dataset_slice": "pairwise three-drug format",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Accuracy",
          "Self-consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/ddi_identification_table3.csv; server/data/benchmark.json",
        "notes": "Table 3 row: DDI 3-Drug Combo.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 87,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 94,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 85,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 82,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 84,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 94,
            "score_status": "published_table"
          }
        ]
      },
      {
        "task_name": "DDI Multi-Drug",
        "dataset_slice": "listwise 4-6 drug format",
        "metrics": [
          "Precision",
          "Recall",
          "F1-score",
          "Accuracy",
          "Self-consistency"
        ],
        "source_status": "published_table",
        "canonical_score_source": "docs/appendices/sources/ddi_identification_table3.csv; server/data/benchmark.json",
        "notes": "Table 3 row: DDI Multi-Drug.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 71,
            "score_status": "published_table"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 88,
            "score_status": "published_table"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": 80,
            "score_status": "published_table"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 69,
            "score_status": "published_table"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 76,
            "score_status": "published_table"
          },
          {
            "model": "DrugGPT",
            "score_percent": 93,
            "score_status": "published_table"
          }
        ]
      }
    ]
  },
  {
    "paper_id": "medmatch",
    "title": "MedMatch: a first step for the automation of large language model performance benchmarking for medication-related tasks",
    "domain": "Medication order structuring and route selection",
    "doi": "10.64898/2026.01.13.26343949",
    "pmc": "PMC12870651",
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12870651/",
    "github": "https://github.com/AIChemist-Lab/MedMatch",
    "dashboard_metric": "MedMatch",
    "dataset_scope": "100 clinician-annotated medication prompts; JSON slot filling and route selection",
    "source_files": [
      "docs/appendices/sources/entity_accuracy_table.csv",
      "docs/appendices/sources/route_accuracy_table.csv",
      "server/data/benchmark.json"
    ],
    "notes": "Scores are source-derived aggregates from MedMatch entity and route source tables; DrugGPT and MedGemma-27B are not reported.",
    "model_scores": [
      {
        "model": "GPT-4o-mini",
        "score_fraction": 0.904,
        "score_percent": 90.4,
        "score_status": "source_derived_aggregate"
      },
      {
        "model": "GPT-5 Chat",
        "score_fraction": 0.916,
        "score_percent": 91.6,
        "score_status": "source_derived_aggregate"
      },
      {
        "model": "MedGemma-27B",
        "score_fraction": null,
        "score_percent": null,
        "score_status": "not_reported_for_model"
      },
      {
        "model": "Gemma 3 27B",
        "score_fraction": 0.905,
        "score_percent": 90.5,
        "score_status": "source_derived_aggregate"
      },
      {
        "model": "Llama 3.3 70B",
        "score_fraction": 0.868,
        "score_percent": 86.8,
        "score_status": "source_derived_aggregate"
      },
      {
        "model": "Qwen3 32B",
        "score_fraction": 0.901,
        "score_percent": 90.1,
        "score_status": "source_derived_aggregate"
      },
      {
        "model": "DrugGPT",
        "score_fraction": null,
        "score_percent": null,
        "score_status": "not_reported_for_model"
      }
    ],
    "tasks": [
      {
        "task_name": "MedMatch (Oral Solid)",
        "dataset_slice": "n=40",
        "metrics": [
          "MedMatch score (exact field match)",
          "Micro-F1"
        ],
        "source_status": "source_derived_aggregate",
        "canonical_score_source": "docs/appendices/sources/entity_accuracy_table.csv; docs/appendices/sources/route_accuracy_table.csv; server/data/benchmark.json",
        "notes": "Entity accuracy category: Oral Solid.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 94,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 95,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 94,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 95,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 95,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      },
      {
        "task_name": "MedMatch (Oral Liq)",
        "dataset_slice": "n=10",
        "metrics": [
          "MedMatch score (exact field match)",
          "Micro-F1"
        ],
        "source_status": "source_derived_aggregate",
        "canonical_score_source": "docs/appendices/sources/entity_accuracy_table.csv; docs/appendices/sources/route_accuracy_table.csv; server/data/benchmark.json",
        "notes": "Entity accuracy category: Oral Liq.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 91,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 90,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 91,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 84,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 89,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      },
      {
        "task_name": "MedMatch (IV Intermit)",
        "dataset_slice": "n=17",
        "metrics": [
          "MedMatch score (exact field match)",
          "Micro-F1"
        ],
        "source_status": "source_derived_aggregate",
        "canonical_score_source": "docs/appendices/sources/entity_accuracy_table.csv; docs/appendices/sources/route_accuracy_table.csv; server/data/benchmark.json",
        "notes": "Entity accuracy category: IV Intermit.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 97,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 96,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 97,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 94,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 96,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      },
      {
        "task_name": "MedMatch (IV Push)",
        "dataset_slice": "n=17",
        "metrics": [
          "MedMatch score (exact field match)",
          "Micro-F1"
        ],
        "source_status": "source_derived_aggregate",
        "canonical_score_source": "docs/appendices/sources/entity_accuracy_table.csv; docs/appendices/sources/route_accuracy_table.csv; server/data/benchmark.json",
        "notes": "Entity accuracy category: IV Push.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 96,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 95,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 95,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 94,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 95,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      },
      {
        "task_name": "MedMatch (Continuous Titrate)",
        "dataset_slice": "n=11",
        "metrics": [
          "MedMatch score (exact field match)",
          "Micro-F1"
        ],
        "source_status": "source_derived_aggregate",
        "canonical_score_source": "docs/appendices/sources/entity_accuracy_table.csv; docs/appendices/sources/route_accuracy_table.csv; server/data/benchmark.json",
        "notes": "Entity accuracy category: Continuous Titrate.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 84,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 86,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 88,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 78,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 88,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      },
      {
        "task_name": "MedMatch (Continuous Non-Titrate)",
        "dataset_slice": "n=6",
        "metrics": [
          "MedMatch score (exact field match)",
          "Micro-F1"
        ],
        "source_status": "source_derived_aggregate",
        "canonical_score_source": "docs/appendices/sources/entity_accuracy_table.csv; docs/appendices/sources/route_accuracy_table.csv; server/data/benchmark.json",
        "notes": "Entity accuracy category: Continuous Non-Titrate.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 80,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 81,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 83,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 81,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 83,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      },
      {
        "task_name": "MedMatch Route Selection",
        "dataset_slice": "route categories",
        "metrics": [
          "Route accuracy",
          "MedMatch score"
        ],
        "source_status": "source_derived_aggregate",
        "canonical_score_source": "docs/appendices/sources/entity_accuracy_table.csv; docs/appendices/sources/route_accuracy_table.csv; server/data/benchmark.json",
        "notes": "Route accuracy table mean across route categories.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 90,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": 98,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 86,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 81,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 84,
            "score_status": "source_derived_aggregate"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      }
    ]
  },
  {
    "paper_id": "drug-or-pokemon",
    "title": "Drug or Pokemon? Large language model performance in identification of fabricated medications",
    "domain": "Adversarial fabricated-medication safety",
    "doi": "10.64898/2026.01.12.26343930",
    "pmc": "PMC12870567",
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12870567/",
    "github": "https://github.com/AIChemist-Lab/Pokemon-Drugs-Names",
    "dashboard_metric": "Drug or Pok\u00e9mon?",
    "dataset_scope": "250 adversarial vignettes with fabricated generic and brand medication names",
    "source_files": [
      "server/scripts/build_benchmark_from_appendices.py embedded PMC Table 2 rates",
      "server/data/benchmark.json"
    ],
    "notes": "Scores are suspicion detected = 100 - default drug-dosing confabulation rate; GPT-5 Chat, MedGemma-27B, and DrugGPT are not reported.",
    "model_scores": [
      {
        "model": "GPT-4o-mini",
        "score_fraction": 0.018,
        "score_percent": 1.8,
        "score_status": "computed_from_pmc_table_2"
      },
      {
        "model": "GPT-5 Chat",
        "score_fraction": null,
        "score_percent": null,
        "score_status": "not_reported_for_model"
      },
      {
        "model": "MedGemma-27B",
        "score_fraction": null,
        "score_percent": null,
        "score_status": "not_reported_for_model"
      },
      {
        "model": "Gemma 3 27B",
        "score_fraction": 0.032,
        "score_percent": 3.2,
        "score_status": "computed_from_pmc_table_2"
      },
      {
        "model": "Llama 3.3 70B",
        "score_fraction": 0.111,
        "score_percent": 11.1,
        "score_status": "computed_from_pmc_table_2"
      },
      {
        "model": "Qwen3 32B",
        "score_fraction": 0.014,
        "score_percent": 1.4,
        "score_status": "computed_from_pmc_table_2"
      },
      {
        "model": "DrugGPT",
        "score_fraction": null,
        "score_percent": null,
        "score_status": "not_reported_for_model"
      }
    ],
    "tasks": [
      {
        "task_name": "Pok\u00e9mon (Generic)",
        "dataset_slice": "250 generic-name poisoned vignettes",
        "metrics": [
          "LLM-as-a-judge",
          "Confabulation rate"
        ],
        "source_status": "computed_from_pmc_table_2",
        "canonical_score_source": "server/scripts/build_benchmark_from_appendices.py embedded PMC Table 2 rates; server/data/benchmark.json",
        "notes": "Suspicion detected = 100 - default dosing confabulation rate.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 2,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 4,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 14,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 2,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      },
      {
        "task_name": "Pok\u00e9mon (Brand)",
        "dataset_slice": "250 brand-name poisoned vignettes",
        "metrics": [
          "LLM-as-a-judge",
          "Confabulation rate"
        ],
        "source_status": "computed_from_pmc_table_2",
        "canonical_score_source": "server/scripts/build_benchmark_from_appendices.py embedded PMC Table 2 rates; server/data/benchmark.json",
        "notes": "Suspicion detected = 100 - default dosing confabulation rate.",
        "model_scores": [
          {
            "model": "GPT-4o-mini",
            "score_percent": 1,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "GPT-5 Chat",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "MedGemma-27B",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          },
          {
            "model": "Gemma 3 27B",
            "score_percent": 2,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "Llama 3.3 70B",
            "score_percent": 8,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "Qwen3 32B",
            "score_percent": 1,
            "score_status": "computed_from_pmc_table_2"
          },
          {
            "model": "DrugGPT",
            "score_percent": null,
            "score_status": "not_reported_for_model"
          }
        ]
      }
    ]
  }
]