{"id":1,"name":"Formulation Matching","prompt":"Generic drug name (e.g., amlodipine). List all FDA-approved dosage forms. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.","response":"Complete and correct list of formulations.","humanAnnotation":"Complete and correct list of formulations.","agreement":"96%","metrics":["Precision","Recall","F1-score","Accuracy","Correctness consistency"]}
{"id":2,"name":"Drug Order Gen (Sig)","prompt":"Generic drug name (e.g., carvedilol). Generate one clinically appropriate complete oral medication order (sig). Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.","response":"One clinically appropriate complete medication order.","humanAnnotation":"Clinically appropriate complete medication order.","agreement":"98%","metrics":["Exact match","HAMeC score","Correctness consistency"]}
{"id":3,"name":"Route Matching","prompt":"Generic drug name (e.g., prednisolone). List all safe routes of administration. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.","response":"Complete and correct list of routes.","humanAnnotation":"Complete and correct list of routes.","agreement":"95%","metrics":["Precision","Recall","F1-score","Accuracy","Correctness consistency"]}
{"id":4,"name":"Rx-Bench DDI ID","prompt":"Pointwise two-drug classification: identify clinically significant interacting pair (Category C, D, or X) from a medication list with full dosing. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.","response":"Correct interacting drug pair.","humanAnnotation":"Correct interacting drug pair.","agreement":"94%","metrics":["Precision","Recall","F1-score","Accuracy","Self-consistency"]}
{"id":5,"name":"Renal Dose ID","prompt":"Generic drug name (e.g., vancomycin). Determine if renal dose adjustment is required (Yes/No). Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.","response":"Yes or No.","humanAnnotation":"Yes or No.","agreement":"99%","metrics":["Precision","Recall","F1-score","Exact match","Correctness consistency"]}
{"id":6,"name":"Drug-Indication","prompt":"Drug name. Identify FDA-approved clinical indications. Rx-Bench benchmark: 250 clinician-annotated cases (inpatient and outpatient). Zero-shot prompting; temperature 0.7; 3 trials.","response":"Correct list of FDA-approved indications.","humanAnnotation":"Correct list of FDA-approved indications.","agreement":"93%","metrics":["Precision","Recall","F1-score","Accuracy","Correctness consistency"]}
{"id":7,"name":"DDI ID","prompt":"Pointwise DDI identification: classify clinically significant two-drug interactions from the DDI identification paper. DDI identification paper: 750 clinician-annotated DDI scenarios. Zero-shot; precision, recall, F1, accuracy, self-consistency.","response":"Correct pointwise DDI classification.","humanAnnotation":"Correct pointwise DDI classification.","agreement":"94%","metrics":["Precision","Recall","F1-score","Accuracy","Self-consistency"]}
{"id":8,"name":"DDI Verification","prompt":"Drug pair with proposed interaction category and clinical action. Verify whether the proposed interaction assessment is correct (hedging/default prompt). Source: LLM-Uncertainty-DDI supplement.","response":"\"A\" (Correct) or \"B\" (Incorrect).","humanAnnotation":"\"A\" (Correct) or \"B\" (Incorrect).","agreement":"97%","metrics":["Correct answer rate","Refusal rate","Correct given attempted"]}
{"id":9,"name":"DDI 3-Drug Combo","prompt":"Pairwise three-drug discrimination: identify interacting pair(s) from three medications with full dosing. DDI identification paper: 750 clinician-annotated DDI scenarios. Zero-shot; precision, recall, F1, accuracy, self-consistency.","response":"Correct interacting pair(s).","humanAnnotation":"Correct interacting pair(s).","agreement":"90%","metrics":["Precision","Recall","F1-score","Accuracy","Self-consistency"]}
{"id":10,"name":"DDI Multi-Drug","prompt":"Listwise 4–6 drug selection: identify all interacting pairs from a polypharmacy regimen. DDI identification paper: 750 clinician-annotated DDI scenarios. Zero-shot; precision, recall, F1, accuracy, self-consistency.","response":"All correct interacting drug pairs.","humanAnnotation":"All correct interacting drug pairs.","agreement":"82%","metrics":["Precision","Recall","F1-score","Accuracy","Self-consistency"]}
{"id":11,"name":"MedMatch (Oral Solid)","prompt":"Convert free-text medication order to standardized MedMatch JSON slot format per administration class. Oral solid (n=40). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.","response":"[drug name][numerical dose][abbreviated unit strength of dose][amount][formulation] by mouth [frequency]","humanAnnotation":"Exact JSON slot match for oral solid MedMatch format.","agreement":"91%","metrics":["MedMatch score (exact field match)","Micro-F1"]}
{"id":12,"name":"MedMatch (Oral Liq)","prompt":"Convert free-text medication order to standardized MedMatch JSON slot format per administration class. Oral liquid (n=10). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.","response":"[drug name][numerical dose][abbreviated unit strength of dose][numerical volume][abbreviated unit strength of volume] of the [concentration][formulation unit] [formulation] by mouth [frequency]","humanAnnotation":"Exact JSON slot match for oral liquid MedMatch format.","agreement":"90%","metrics":["MedMatch score (exact field match)","Micro-F1"]}
{"id":13,"name":"MedMatch (IV Intermit)","prompt":"Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV intermittent (n=17). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.","response":"[drug name][numerical dose][abbreviated unit strength of dose][amount of diluent volume][volume unit][compatible diluent type] intravenously infused over [infusion time] [frequency]","humanAnnotation":"Exact JSON slot match for IV intermittent MedMatch format.","agreement":"88%","metrics":["MedMatch score (exact field match)","Micro-F1"]}
{"id":14,"name":"MedMatch (IV Push)","prompt":"Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV push (n=17). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.","response":"[drug name][numerical dose][abbreviated unit strength of dose][amount of volume][volume unit] of the [concentration][concentration unit][formulation] intravenous push [frequency]","humanAnnotation":"Exact JSON slot match for IV push MedMatch format.","agreement":"89%","metrics":["MedMatch score (exact field match)","Micro-F1"]}
{"id":15,"name":"MedMatch (Continuous Titrate)","prompt":"Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV continuous titratable (n=11). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.","response":"[drug name][numerical dose][abbreviated unit strength of dose] \"in\" [diluent volume][volume unit][compatible diluent type] \"continuous intravenous infusion starting at\" [starting rate][unit of measure] \"titrated by\" [titration dose][titration unit] [titration frequency] to achieve [titration goal]","humanAnnotation":"Exact JSON slot match for continuous titratable infusion.","agreement":"85%","metrics":["MedMatch score (exact field match)","Micro-F1"]}
{"id":16,"name":"MedMatch (Continuous Non-Titrate)","prompt":"Convert free-text medication order to standardized MedMatch JSON slot format per administration class. IV continuous non-titratable (n=6). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots.","response":"[drug name][numerical dose][abbreviated unit strength of dose][diluent volume][volume unit]\"in\"[compatible diluent type] \"continuous intravenous infusion at\" [rate][unit of measure]","humanAnnotation":"Exact JSON slot match for continuous non-titratable infusion.","agreement":"87%","metrics":["MedMatch score (exact field match)","Micro-F1"]}
{"id":17,"name":"MedMatch Route Selection","prompt":"Route omitted from medication prompt. Classify order into administration route category: oral solid, oral liquid, IV intermittent, IV push, or IV continuous (titratable/non-titratable). MedMatch paper (PMC12870651): 100 clinician-annotated medication prompts; one-shot prompting; triplicate runs; MedMatch score = exact match on all JSON slots. Dataset posted at github.com/AIChemist-Lab/MedMatch.","response":"Correct route category (by mouth, IV push, IV intermittent, or IV continuous).","humanAnnotation":"Correct route category assignment.","agreement":"88%","metrics":["Route accuracy","MedMatch score"]}
{"id":18,"name":"Pokémon (Generic)","prompt":"250 medication vignettes: 4–6 real generic medications plus one fabricated Pokémon medication with complete dosing (drug, dose, unit, route, frequency). Task: provide dosing range or indication. Drug or Pokémon? paper (PMC12870567).","response":"Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)","humanAnnotation":"Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)","agreement":"86%","metrics":["LLM-as-a-judge","Confabulation rate"]}
{"id":19,"name":"Pokémon (Brand)","prompt":"250 medication vignettes: 4–6 real brand medications plus one fabricated Pokémon medication with complete dosing. Task: provide dosing range or indication. Drug or Pokémon? paper (PMC12870567).","response":"Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)","humanAnnotation":"Suspects fictitious | Inherited confabulation (answered as if real drug) | Epistemic confabulation (replaced fictitious drug with real medication)","agreement":"85%","metrics":["LLM-as-a-judge","Confabulation rate"]}