server/data/benchmark.json

{
  "models": [
    {
      "name": "GPT-4o-mini",
      "type": "Standard",
      "provider": "OpenAI",
      "access": "API",
      "winRate": 0.602,
      "costPer1mTokens": "$0.15",
      "latency": "2.1s",
      "isCustom": false
    },
    {
      "name": "GPT-5 Chat",
      "type": "Reasoning",
      "provider": "OpenAI",
      "access": "API",
      "winRate": 0.88,
      "costPer1mTokens": "$1.10",
      "latency": "8.2s",
      "isCustom": false
    },
    {
      "name": "MedGemma-27B",
      "type": "Medical",
      "provider": "Google",
      "access": "Open Weights",
      "winRate": 0.718,
      "costPer1mTokens": "$0.15",
      "latency": "1.8s",
      "isCustom": false
    },
    {
      "name": "Gemma 3 27B",
      "type": "Standard",
      "provider": "Google",
      "access": "Open Weights",
      "winRate": 0.469,
      "costPer1mTokens": "$0.15",
      "latency": "1.8s",
      "isCustom": false
    },
    {
      "name": "Llama 3.3 70B",
      "type": "Standard",
      "provider": "Meta",
      "access": "Open Weights",
      "winRate": 0.613,
      "costPer1mTokens": "$0.70",
      "latency": "3.2s",
      "isCustom": false
    },
    {
      "name": "Qwen3 32B",
      "type": "Standard",
      "provider": "Alibaba",
      "access": "Open Weights",
      "winRate": 0.6,
      "costPer1mTokens": "$0.20",
      "latency": "2.5s",
      "isCustom": false
    },
    {
      "name": "DrugGPT",
      "type": "Standard",
      "provider": "DrugGPT",
      "access": "Specialized",
      "winRate": 0.787,
      "costPer1mTokens": "N/A",
      "latency": "N/A",
      "isCustom": false
    }
  ],
  "benchmarkResults": [
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Formulation Matching",
      "earned": 53.1,
      "failed": 46.9
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Drug Order Gen (Sig)",
      "earned": 83.2,
      "failed": 16.8
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Route Matching",
      "earned": 67.7,
      "failed": 32.3
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Rx-Bench DDI ID",
      "earned": 70.4,
      "failed": 29.6
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Renal Dose ID",
      "earned": 83.3,
      "failed": 16.7
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Drug-Indication",
      "earned": 96.5,
      "failed": 3.5
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "MedMatch Route Selection",
      "earned": 90.1,
      "failed": 9.9
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "DDI ID",
      "earned": 61.0,
      "failed": 39.0
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "DDI 3-Drug Combo",
      "earned": 86.6,
      "failed": 13.4
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "DDI Multi-Drug",
      "earned": 71.3,
      "failed": 28.7
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "DDI Verification",
      "earned": 65.0,
      "failed": 35.0
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "MedMatch (Oral Solid)",
      "earned": 94.2,
      "failed": 5.8
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "MedMatch (Oral Liq)",
      "earned": 90.7,
      "failed": 9.3
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "MedMatch (IV Intermit)",
      "earned": 97.3,
      "failed": 2.7
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "MedMatch (IV Push)",
      "earned": 96.5,
      "failed": 3.5
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "MedMatch (Continuous Titrate)",
      "earned": 84.3,
      "failed": 15.7
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "MedMatch (Continuous Non-Titrate)",
      "earned": 79.9,
      "failed": 20.1
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Pok\u00e9mon (Generic)",
      "earned": 2.3,
      "failed": 97.7
    },
    {
      "modelName": "GPT-4o-mini",
      "taskName": "Pok\u00e9mon (Brand)",
      "earned": 1.2,
      "failed": 98.8
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "Formulation Matching",
      "earned": 73.0,
      "failed": 27.0
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "Drug Order Gen (Sig)",
      "earned": 90.0,
      "failed": 10.0
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "Route Matching",
      "earned": 75.2,
      "failed": 24.8
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "Rx-Bench DDI ID",
      "earned": 84.8,
      "failed": 15.2
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "Renal Dose ID",
      "earned": 87.4,
      "failed": 12.6
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "Drug-Indication",
      "earned": 99.3,
      "failed": 0.7
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "MedMatch Route Selection",
      "earned": 98.4,
      "failed": 1.6
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "DDI ID",
      "earned": 80.6,
      "failed": 19.4
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "DDI 3-Drug Combo",
      "earned": 93.6,
      "failed": 6.4
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "DDI Multi-Drug",
      "earned": 88.4,
      "failed": 11.6
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "DDI Verification",
      "earned": 83.3,
      "failed": 16.7
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "MedMatch (Oral Solid)",
      "earned": 95.0,
      "failed": 5.0
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "MedMatch (Oral Liq)",
      "earned": 90.0,
      "failed": 10.0
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "MedMatch (IV Intermit)",
      "earned": 96.3,
      "failed": 3.7
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "MedMatch (IV Push)",
      "earned": 94.8,
      "failed": 5.2
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "MedMatch (Continuous Titrate)",
      "earned": 85.8,
      "failed": 14.2
    },
    {
      "modelName": "GPT-5 Chat",
      "taskName": "MedMatch (Continuous Non-Titrate)",
      "earned": 80.6,
      "failed": 19.4
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "Formulation Matching",
      "earned": 40.9,
      "failed": 59.1
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "Drug Order Gen (Sig)",
      "earned": 81.2,
      "failed": 18.8
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "Route Matching",
      "earned": 69.1,
      "failed": 30.9
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "Rx-Bench DDI ID",
      "earned": 66.5,
      "failed": 33.5
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "Renal Dose ID",
      "earned": 76.9,
      "failed": 23.1
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "Drug-Indication",
      "earned": 96.9,
      "failed": 3.1
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "DDI ID",
      "earned": 50.1,
      "failed": 49.9
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "DDI 3-Drug Combo",
      "earned": 84.9,
      "failed": 15.1
    },
    {
      "modelName": "MedGemma-27B",
      "taskName": "DDI Multi-Drug",
      "earned": 80.0,
      "failed": 20.0
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "MedMatch Route Selection",
      "earned": 85.8,
      "failed": 14.2
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "DDI Verification",
      "earned": 65.9,
      "failed": 34.1
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "MedMatch (Oral Solid)",
      "earned": 93.6,
      "failed": 6.4
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "MedMatch (Oral Liq)",
      "earned": 91.0,
      "failed": 9.0
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "MedMatch (IV Intermit)",
      "earned": 96.8,
      "failed": 3.2
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "MedMatch (IV Push)",
      "earned": 95.4,
      "failed": 4.6
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "MedMatch (Continuous Titrate)",
      "earned": 87.9,
      "failed": 12.1
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "MedMatch (Continuous Non-Titrate)",
      "earned": 83.3,
      "failed": 16.7
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "Pok\u00e9mon (Generic)",
      "earned": 4.1,
      "failed": 95.9
    },
    {
      "modelName": "Gemma 3 27B",
      "taskName": "Pok\u00e9mon (Brand)",
      "earned": 2.3,
      "failed": 97.7
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Formulation Matching",
      "earned": 54.0,
      "failed": 46.0
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Drug Order Gen (Sig)",
      "earned": 88.0,
      "failed": 12.0
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Route Matching",
      "earned": 74.3,
      "failed": 25.7
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Rx-Bench DDI ID",
      "earned": 66.7,
      "failed": 33.3
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Renal Dose ID",
      "earned": 83.2,
      "failed": 16.8
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Drug-Indication",
      "earned": 97.6,
      "failed": 2.4
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "MedMatch Route Selection",
      "earned": 81.2,
      "failed": 18.8
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "DDI ID",
      "earned": 59.8,
      "failed": 40.2
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "DDI 3-Drug Combo",
      "earned": 81.5,
      "failed": 18.5
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "DDI Multi-Drug",
      "earned": 68.6,
      "failed": 31.4
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "DDI Verification",
      "earned": 73.9,
      "failed": 26.1
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "MedMatch (Oral Solid)",
      "earned": 94.6,
      "failed": 5.4
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "MedMatch (Oral Liq)",
      "earned": 84.3,
      "failed": 15.7
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "MedMatch (IV Intermit)",
      "earned": 94.1,
      "failed": 5.9
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "MedMatch (IV Push)",
      "earned": 94.5,
      "failed": 5.5
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "MedMatch (Continuous Titrate)",
      "earned": 77.5,
      "failed": 22.5
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "MedMatch (Continuous Non-Titrate)",
      "earned": 81.2,
      "failed": 18.8
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Pok\u00e9mon (Generic)",
      "earned": 14.0,
      "failed": 86.0
    },
    {
      "modelName": "Llama 3.3 70B",
      "taskName": "Pok\u00e9mon (Brand)",
      "earned": 8.1,
      "failed": 91.9
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Formulation Matching",
      "earned": 36.6,
      "failed": 63.4
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Drug Order Gen (Sig)",
      "earned": 79.6,
      "failed": 20.4
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Route Matching",
      "earned": 71.6,
      "failed": 28.4
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Rx-Bench DDI ID",
      "earned": 75.6,
      "failed": 24.4
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Renal Dose ID",
      "earned": 79.7,
      "failed": 20.3
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Drug-Indication",
      "earned": 94.0,
      "failed": 6.0
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "MedMatch Route Selection",
      "earned": 84.3,
      "failed": 15.7
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "DDI ID",
      "earned": 66.7,
      "failed": 33.3
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "DDI 3-Drug Combo",
      "earned": 84.0,
      "failed": 16.0
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "DDI Multi-Drug",
      "earned": 76.1,
      "failed": 23.9
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "DDI Verification",
      "earned": 59.4,
      "failed": 40.6
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "MedMatch (Oral Solid)",
      "earned": 95.0,
      "failed": 5.0
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "MedMatch (Oral Liq)",
      "earned": 89.0,
      "failed": 11.0
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "MedMatch (IV Intermit)",
      "earned": 95.8,
      "failed": 4.2
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "MedMatch (IV Push)",
      "earned": 95.2,
      "failed": 4.8
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "MedMatch (Continuous Titrate)",
      "earned": 88.1,
      "failed": 11.9
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "MedMatch (Continuous Non-Titrate)",
      "earned": 83.3,
      "failed": 16.7
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Pok\u00e9mon (Generic)",
      "earned": 1.6,
      "failed": 98.4
    },
    {
      "modelName": "Qwen3 32B",
      "taskName": "Pok\u00e9mon (Brand)",
      "earned": 1.2,
      "failed": 98.8
    },
    {
      "modelName": "DrugGPT",
      "taskName": "Formulation Matching",
      "earned": 63.4,
      "failed": 36.6
    },
    {
      "modelName": "DrugGPT",
      "taskName": "Drug Order Gen (Sig)",
      "earned": 80.4,
      "failed": 19.6
    },
    {
      "modelName": "DrugGPT",
      "taskName": "Route Matching",
      "earned": 74.0,
      "failed": 26.0
    },
    {
      "modelName": "DrugGPT",
      "taskName": "Rx-Bench DDI ID",
      "earned": 77.7,
      "failed": 22.3
    },
    {
      "modelName": "DrugGPT",
      "taskName": "Renal Dose ID",
      "earned": 77.9,
      "failed": 22.1
    },
    {
      "modelName": "DrugGPT",
      "taskName": "Drug-Indication",
      "earned": 98.4,
      "failed": 1.6
    },
    {
      "modelName": "DrugGPT",
      "taskName": "DDI ID",
      "earned": 50.1,
      "failed": 49.9
    },
    {
      "modelName": "DrugGPT",
      "taskName": "DDI 3-Drug Combo",
      "earned": 93.7,
      "failed": 6.3
    },
    {
      "modelName": "DrugGPT",
      "taskName": "DDI Multi-Drug",
      "earned": 92.6,
      "failed": 7.4
    }
  ],
  "leaderboardScores": [
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Mean Win Rate",
      "tab": "Accuracy",
      "value": "0.602"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Rx-Bench (CMM)",
      "tab": "Accuracy",
      "value": "0.757"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "DDI Identification",
      "tab": "Accuracy",
      "value": "0.730"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "MedMatch",
      "tab": "Accuracy",
      "value": "0.904"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Drug or Pok\u00e9mon?",
      "tab": "Accuracy",
      "value": "0.018"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Cost (per 1M tokens)",
      "tab": "Efficiency",
      "value": "$0.15"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Latency (s / request)",
      "tab": "Efficiency",
      "value": "2.1s"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Provider",
      "tab": "General information",
      "value": "OpenAI"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Access",
      "tab": "General information",
      "value": "API"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Model Type",
      "tab": "General information",
      "value": "Standard"
    },
    {
      "modelName": "GPT-4o-mini",
      "metricName": "Source Coverage",
      "tab": "General information",
      "value": "4/4"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Mean Win Rate",
      "tab": "Accuracy",
      "value": "0.880"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Rx-Bench (CMM)",
      "tab": "Accuracy",
      "value": "0.850"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "DDI Identification",
      "tab": "Accuracy",
      "value": "0.875"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "MedMatch",
      "tab": "Accuracy",
      "value": "0.916"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Drug or Pok\u00e9mon?",
      "tab": "Accuracy",
      "value": "N/A"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Cost (per 1M tokens)",
      "tab": "Efficiency",
      "value": "$1.10"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Latency (s / request)",
      "tab": "Efficiency",
      "value": "8.2s"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Provider",
      "tab": "General information",
      "value": "OpenAI"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Access",
      "tab": "General information",
      "value": "API"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Model Type",
      "tab": "General information",
      "value": "Reasoning"
    },
    {
      "modelName": "GPT-5 Chat",
      "metricName": "Source Coverage",
      "tab": "General information",
      "value": "3/4"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Mean Win Rate",
      "tab": "Accuracy",
      "value": "0.718"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Rx-Bench (CMM)",
      "tab": "Accuracy",
      "value": "0.719"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "DDI Identification",
      "tab": "Accuracy",
      "value": "0.717"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "MedMatch",
      "tab": "Accuracy",
      "value": "N/A"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Drug or Pok\u00e9mon?",
      "tab": "Accuracy",
      "value": "N/A"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Cost (per 1M tokens)",
      "tab": "Efficiency",
      "value": "$0.15"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Latency (s / request)",
      "tab": "Efficiency",
      "value": "1.8s"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Provider",
      "tab": "General information",
      "value": "Google"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Access",
      "tab": "General information",
      "value": "Open Weights"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Model Type",
      "tab": "General information",
      "value": "Medical"
    },
    {
      "modelName": "MedGemma-27B",
      "metricName": "Source Coverage",
      "tab": "General information",
      "value": "2/4"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Mean Win Rate",
      "tab": "Accuracy",
      "value": "0.469"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Rx-Bench (CMM)",
      "tab": "Accuracy",
      "value": "N/A"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "DDI Identification",
      "tab": "Accuracy",
      "value": "N/A"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "MedMatch",
      "tab": "Accuracy",
      "value": "0.905"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Drug or Pok\u00e9mon?",
      "tab": "Accuracy",
      "value": "0.032"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Cost (per 1M tokens)",
      "tab": "Efficiency",
      "value": "$0.15"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Latency (s / request)",
      "tab": "Efficiency",
      "value": "1.8s"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Provider",
      "tab": "General information",
      "value": "Google"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Access",
      "tab": "General information",
      "value": "Open Weights"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Model Type",
      "tab": "General information",
      "value": "Standard"
    },
    {
      "modelName": "Gemma 3 27B",
      "metricName": "Source Coverage",
      "tab": "General information",
      "value": "2/4"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Mean Win Rate",
      "tab": "Accuracy",
      "value": "0.613"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Rx-Bench (CMM)",
      "tab": "Accuracy",
      "value": "0.773"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "DDI Identification",
      "tab": "Accuracy",
      "value": "0.700"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "MedMatch",
      "tab": "Accuracy",
      "value": "0.868"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Drug or Pok\u00e9mon?",
      "tab": "Accuracy",
      "value": "0.111"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Cost (per 1M tokens)",
      "tab": "Efficiency",
      "value": "$0.70"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Latency (s / request)",
      "tab": "Efficiency",
      "value": "3.2s"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Provider",
      "tab": "General information",
      "value": "Meta"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Access",
      "tab": "General information",
      "value": "Open Weights"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Model Type",
      "tab": "General information",
      "value": "Standard"
    },
    {
      "modelName": "Llama 3.3 70B",
      "metricName": "Source Coverage",
      "tab": "General information",
      "value": "4/4"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Mean Win Rate",
      "tab": "Accuracy",
      "value": "0.600"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Rx-Bench (CMM)",
      "tab": "Accuracy",
      "value": "0.728"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "DDI Identification",
      "tab": "Accuracy",
      "value": "0.756"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "MedMatch",
      "tab": "Accuracy",
      "value": "0.901"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Drug or Pok\u00e9mon?",
      "tab": "Accuracy",
      "value": "0.014"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Cost (per 1M tokens)",
      "tab": "Efficiency",
      "value": "$0.20"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Latency (s / request)",
      "tab": "Efficiency",
      "value": "2.5s"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Provider",
      "tab": "General information",
      "value": "Alibaba"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Access",
      "tab": "General information",
      "value": "Open Weights"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Model Type",
      "tab": "General information",
      "value": "Standard"
    },
    {
      "modelName": "Qwen3 32B",
      "metricName": "Source Coverage",
      "tab": "General information",
      "value": "4/4"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Mean Win Rate",
      "tab": "Accuracy",
      "value": "0.787"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Rx-Bench (CMM)",
      "tab": "Accuracy",
      "value": "0.786"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "DDI Identification",
      "tab": "Accuracy",
      "value": "0.788"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "MedMatch",
      "tab": "Accuracy",
      "value": "N/A"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Drug or Pok\u00e9mon?",
      "tab": "Accuracy",
      "value": "N/A"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Cost (per 1M tokens)",
      "tab": "Efficiency",
      "value": "N/A"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Latency (s / request)",
      "tab": "Efficiency",
      "value": "N/A"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Provider",
      "tab": "General information",
      "value": "DrugGPT"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Access",
      "tab": "General information",
      "value": "Specialized"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Model Type",
      "tab": "General information",
      "value": "Standard"
    },
    {
      "modelName": "DrugGPT",
      "metricName": "Source Coverage",
      "tab": "General information",
      "value": "2/4"
    }
  ],
  "taskDefinitions": [
    {
      "name": "Formulation Matching",
      "prompt": "Generic drug name (e.g., amlodipine). List all FDA-approved dosage forms. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.",
      "response": "Complete and correct list of formulations.",
      "humanAnnotation": "Complete and correct list of formulations.",
      "agreement": "96%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Accuracy",
        "Correctness consistency"
      ]
    },
    {
      "name": "Drug Order Gen (Sig)",
      "prompt": "Generic drug name (e.g., carvedilol). Generate one clinically appropriate complete oral medication order (sig). Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.",
      "response": "One clinically appropriate complete medication order.",
      "humanAnnotation": "Clinically appropriate complete medication order.",
      "agreement": "98%",
      "metrics": [
        "Exact match",
        "HAMeC score",
        "Correctness consistency"
      ]
    },
    {
      "name": "Route Matching",
      "prompt": "Generic drug name (e.g., prednisolone). List all safe routes of administration. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.",
      "response": "Complete and correct list of routes.",
      "humanAnnotation": "Complete and correct list of routes.",
      "agreement": "95%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Accuracy",
        "Correctness consistency"
      ]
    },
    {
      "name": "Rx-Bench DDI ID",
      "prompt": "Pointwise two-drug classification: identify clinically significant interacting pair (Category C, D, or X) from a medication list with full dosing. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.",
      "response": "Correct interacting drug pair.",
      "humanAnnotation": "Correct interacting drug pair.",
      "agreement": "94%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Accuracy",
        "Self-consistency"
      ]
    },
    {
      "name": "Renal Dose ID",
      "prompt": "Generic drug name (e.g., vancomycin). Determine if renal dose adjustment is required (Yes/No). Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.",
      "response": "Yes or No.",
      "humanAnnotation": "Yes or No.",
      "agreement": "99%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Exact match",
        "Correctness consistency"
      ]
    },
    {
      "name": "Drug-Indication",
      "prompt": "Drug name. Identify FDA-approved clinical indications. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.",
      "response": "Correct list of FDA-approved indications.",
      "humanAnnotation": "Correct list of FDA-approved indications.",
      "agreement": "93%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Accuracy",
        "Correctness consistency"
      ]
    },
    {
      "name": "DDI ID",
      "prompt": "Pointwise DDI identification: classify clinically significant two-drug interactions from the DDI identification paper. DDI identification paper: 750 clinician-annotated DDI scenarios. Zero-shot; precision, recall, F1, accuracy, self-consistency.",
      "response": "Correct pointwise DDI classification.",
      "humanAnnotation": "Correct pointwise DDI classification.",
      "agreement": "94%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Accuracy",
        "Self-consistency"
      ]
    },
    {
      "name": "DDI Verification",
      "prompt": "Drug pair with proposed interaction category and clinical action. Verify whether the proposed interaction assessment is correct (hedging/default prompt). Source: LLM-Uncertainty-DDI supplement.",
      "response": "\"A\" (Correct) or \"B\" (Incorrect).",
      "humanAnnotation": "\"A\" (Correct) or \"B\" (Incorrect).",
      "agreement": "97%",
      "metrics": [
        "Correct answer rate",
        "Refusal rate",
        "Correct given attempted"
      ]
    },
    {
      "name": "DDI 3-Drug Combo",
      "prompt": "Pairwise three-drug discrimination: identify interacting pair(s) from three medications with full dosing. DDI identification paper: 750 clinician-annotated DDI scenarios. Zero-shot; precision, recall, F1, accuracy, self-consistency.",
      "response": "Correct interacting pair(s).",
      "humanAnnotation": "Correct interacting pair(s).",
      "agreement": "90%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Accuracy",
        "Self-consistency"
      ]
    },
    {
      "name": "DDI Multi-Drug",
      "prompt": "Listwise 4\u20136 drug selection: identify all interacting pairs from a polypharmacy regimen. DDI identification paper: 750 clinician-annotated DDI scenarios. Zero-shot; precision, recall, F1, accuracy, self-consistency.",
      "response": "All correct interacting drug pairs.",
      "humanAnnotation": "All correct interacting drug pairs.",
      "agreement": "82%",
      "metrics": [
        "Precision",
        "Recall",
        "F1-score",
        "Accuracy",
        "Self-consistency"
      ]
    },
    {
      "name": "MedMatch (Oral Solid)",
      "prompt": "Convert free-text medication order to standardized MedMatch JSON slot format per administration class. Oral solid (n=40). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.",
      "response": "[drug name][numerical dose][abbreviated unit strength of dose][amount][formulation] by mouth [frequency]",
      "humanAnnotation": "Exact JSON slot match for oral solid MedMatch format.",
      "agreement": "91%",
      "metrics": [
        "MedMatch score (exact field match)",
        "Micro-F1"
      ]
    },
    {
      "name": "MedMatch (Oral Liq)",
      "prompt": "Convert free-text medication order to standardized MedMatch JSON slot format per administration class. Oral liquid (n=10). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.",
      "response": "[drug name][numerical dose][abbreviated unit strength of dose][numerical volume][abbreviated unit strength of volume] of the [concentration][formulation unit] [formulation] by mouth [frequency]",
      "humanAnnotation": "Exact JSON slot match for oral liquid MedMatch format.",
      "agreement": "90%",
      "metrics": [
        "MedMatch score (exact field match)",
        "Micro-F1"
      ]
    },
    {
      "name": "MedMatch (IV Intermit)",
      "prompt": "Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV intermittent (n=17). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.",
      "response": "[drug name][numerical dose][abbreviated unit strength of dose][amount of diluent volume][volume unit][compatible diluent type] intravenously infused over [infusion time] [frequency]",
      "humanAnnotation": "Exact JSON slot match for IV intermittent MedMatch format.",
      "agreement": "88%",
      "metrics": [
        "MedMatch score (exact field match)",
        "Micro-F1"
      ]
    },
    {
      "name": "MedMatch (IV Push)",
      "prompt": "Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV push (n=17). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.",
      "response": "[drug name][numerical dose][abbreviated unit strength of dose][amount of volume][volume unit] of the [concentration][concentration unit][formulation] intravenous push [frequency]",
      "humanAnnotation": "Exact JSON slot match for IV push MedMatch format.",
      "agreement": "89%",
      "metrics": [
        "MedMatch score (exact field match)",
        "Micro-F1"
      ]
    },
    {
      "name": "MedMatch (Continuous Titrate)",
      "prompt": "Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV continuous titratable (n=11). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.",
      "response": "[drug name][numerical dose][abbreviated unit strength of dose] \"in\" [diluent volume][volume unit][compatible diluent type] \"continuous intravenous infusion starting at\" [starting rate][unit of measure] \"titrated by\" [titration dose][titration unit] [titration frequency] to achieve [titration goal]",
      "humanAnnotation": "Exact JSON slot match for continuous titratable infusion.",
      "agreement": "85%",
      "metrics": [
        "MedMatch score (exact field match)",
        "Micro-F1"
      ]
    },
    {
      "name": "MedMatch (Continuous Non-Titrate)",
      "prompt": "Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV continuous non-titratable (n=6). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.",
      "response": "[drug name][numerical dose][abbreviated unit strength of dose][diluent volume][volume unit]\"in\"[compatible diluent type] \"continuous intravenous infusion at\" [rate][unit of measure]",
      "humanAnnotation": "Exact JSON slot match for continuous non-titratable infusion.",
      "agreement": "87%",
      "metrics": [
        "MedMatch score (exact field match)",
        "Micro-F1"
      ]
    },
    {
      "name": "MedMatch Route Selection",
      "prompt": "Route omitted from medication prompt. Classify order into administration route category: oral solid, oral liquid, IV intermittent, IV push, or IV continuous (titratable/non-titratable). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots. Dataset posted at github.com/AIChemist-Lab/MedMatch.",
      "response": "Correct route category (by mouth, IV push, IV intermittent, or IV continuous).",
      "humanAnnotation": "Correct route category assignment.",
      "agreement": "88%",
      "metrics": [
        "Route accuracy",
        "MedMatch score"
      ]
    },
    {
      "name": "Pok\u00e9mon (Generic)",
      "prompt": "250 medication vignettes: 4\u20136 real generic medications plus one fabricated Pok\u00e9mon medication with complete dosing (drug, dose, unit, route, frequency). Task: provide dosing range or indication. Drug or Pok\u00e9mon? paper (PMC12870567).",
      "response": "Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)",
      "humanAnnotation": "Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)",
      "agreement": "86%",
      "metrics": [
        "LLM-as-a-judge",
        "Confabulation rate"
      ]
    },
    {
      "name": "Pok\u00e9mon (Brand)",
      "prompt": "250 medication vignettes: 4\u20136 real brand medications plus one fabricated Pok\u00e9mon medication with complete dosing. Task: provide dosing range or indication. Drug or Pok\u00e9mon? paper (PMC12870567).",
      "response": "Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)",
      "humanAnnotation": "Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)",
      "agreement": "85%",
      "metrics": [
        "LLM-as-a-judge",
        "Confabulation rate"
      ]
    }
  ],
  "meta": {
    "papers": {
      "rxLlm": {
        "title": "Rx-Bench",
        "doi": "10.64898/2025.12.01.25341004",
        "url": "https://www.medrxiv.org/content/10.64898/2025.12.01.25341004v2"
      },
      "ddiIdentification": {
        "title": "Drug-drug interaction identification using large language models",
        "doi": "10.64898/2025.12.03.25341549",
        "url": "https://www.medrxiv.org/content/10.64898/2025.12.03.25341549v2"
      },
      "pokemon": {
        "title": "Drug or Pok\u00e9mon?",
        "pmc": "PMC12870567",
        "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12870567/"
      },
      "medMatch": {
        "title": "MedMatch: a first step for the automation of large language model performance benchmarking for medication-related tasks",
        "doi": "10.64898/2026.01.13.26343949",
        "pmc": "PMC12870651",
        "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12870651/",
        "github": "https://github.com/AIChemist-Lab/MedMatch"
      }
    },
    "sources": [
      "Rx-Bench submission 4.8.26 / medRxiv 10.64898/2025.12.01.25341004",
      "DDI identification medRxiv 10.64898/2025.12.03.25341549",
      "MedMatch medRxiv 10.64898/2026.01.13.26343949 / PMC12870651",
      "Drug or Pok\u00e9mon? PMC12870567",
      "github.com/AIChemist-Lab/MedMatch, LLM-Uncertainty-DDI appendix tables"
    ],
    "supportedModels": [
      "GPT-4o-mini",
      "GPT-5 Chat",
      "MedGemma-27B",
      "Gemma 3 27B",
      "Llama 3.3 70B",
      "Qwen3 32B",
      "DrugGPT"
    ],
    "scorePolicy": {
      "reportedMean": "Mean Win Rate averages only source-backed paper scores and excludes N/A cells.",
      "sourceCoverage": "Number of primary papers with source-backed performance for the model out of four.",
      "rxLlm": "Rx-Bench (CMM) is the macro mean of the six primary task metrics reported in Rx-Bench Tables 2-3; task-level scores remain available in Scenarios.",
      "pokemon": "Drug or Pok\u00e9mon? scores are suspicion detected = 100 - default-dosing confabulation rate; unreported models are N/A, not zero."
    },
    "note": "Mean Win Rate is the reported mean over source-backed paper scores only. Rx-Bench (CMM) is the macro mean of six primary task metrics from Rx-Bench Tables 2-3; task-level scores remain attached to Scenarios rather than the main leaderboard. MedGemma-27B is listed separately where source tables report MedGemma rather than base Gemma 3 27B. DrugGPT scores now include Rx-Bench and DDI identification source tables, but remain N/A for MedMatch and Drug or Pok\u00e9mon? where not reported. DDI Verification remains a supplemental LLM-Uncertainty-DDI row and is not part of the four-paper reported mean. GPT-5 Chat and DrugGPT were not evaluated in the Drug or Pok\u00e9mon? source table. Cost and Latency are indicative estimates and are not source-backed."
  }
}