| { |
| "Mistral-Nemo-Instruct-2407-NVFP4": { |
| "coqa": { |
| "alias": "coqa", |
| "em,none": 0.5391666666666667, |
| "em_stderr,none": 0.019942406218865946, |
| "f1,none": 0.7182411035342305, |
| "f1_stderr,none": 0.015116252267763605 |
| }, |
| "hellaswag": { |
| "alias": "hellaswag", |
| "acc,none": 0.6186018721370244, |
| "acc_stderr,none": 0.004847372670134442, |
| "acc_norm,none": 0.8084047002589125, |
| "acc_norm_stderr,none": 0.003927519561403822 |
| }, |
| "ifeval": { |
| "alias": "ifeval", |
| "prompt_level_strict_acc,none": 0.3807763401109057, |
| "prompt_level_strict_acc_stderr,none": 0.020895937888190826, |
| "inst_level_strict_acc,none": 0.4712230215827338, |
| "inst_level_strict_acc_stderr,none": "N/A", |
| "prompt_level_loose_acc,none": 0.4602587800369686, |
| "prompt_level_loose_acc_stderr,none": 0.021448501434135032, |
| "inst_level_loose_acc,none": 0.5455635491606715, |
| "inst_level_loose_acc_stderr,none": "N/A" |
| }, |
| "lambada_openai": { |
| "alias": "lambada_openai", |
| "perplexity,none": 3.0228658176330048, |
| "perplexity_stderr,none": 0.05627099222260382, |
| "acc,none": 0.7583931690277508, |
| "acc_stderr,none": 0.0059636738430670555 |
| }, |
| "lambada_openai_cloze_yaml": { |
| "alias": "lambada_openai_cloze_yaml", |
| "perplexity,none": 29.84274462902617, |
| "perplexity_stderr,none": 0.7625202924355495, |
| "acc,none": 0.3122452940034931, |
| "acc_stderr,none": 0.0064561975253284295 |
| }, |
| "lambada_standard": { |
| "alias": "lambada_standard", |
| "perplexity,none": 3.640092189793144, |
| "perplexity_stderr,none": 0.07656519320424794, |
| "acc,none": 0.688530952843004, |
| "acc_stderr,none": 0.006451805320261074 |
| }, |
| "lambada_standard_cloze_yaml": { |
| "alias": "lambada_standard_cloze_yaml", |
| "perplexity,none": 44.843956670053444, |
| "perplexity_stderr,none": 1.1468558717049042, |
| "acc,none": 0.22588783233068116, |
| "acc_stderr,none": 0.005825865294666935 |
| }, |
| "commonsense_qa": { |
| "alias": "commonsense_qa", |
| "acc,none": 0.5773955773955773, |
| "acc_stderr,none": 0.014142423233580207 |
| }, |
| "mmlu": { |
| "acc,none": 0.6324597635664435, |
| "acc_stderr,none": 0.0037954721741336904, |
| "alias": "mmlu" |
| }, |
| "openbookqa": { |
| "alias": "openbookqa", |
| "acc,none": 0.368, |
| "acc_stderr,none": 0.021588982568353548, |
| "acc_norm,none": 0.47, |
| "acc_norm_stderr,none": 0.0223427481925028 |
| }, |
| "winogrande": { |
| "alias": "winogrande", |
| "acc,none": 0.7671665351223362, |
| "acc_stderr,none": 0.011878201073856598 |
| }, |
| "triviaqa": { |
| "alias": "triviaqa", |
| "exact_match,remove_whitespace": 0.595296477931342, |
| "exact_match_stderr,remove_whitespace": 0.003664271290957409 |
| }, |
| "truthfulqa_mc1": { |
| "alias": "truthfulqa_mc1", |
| "acc,none": 0.37821297429620565, |
| "acc_stderr,none": 0.016976335907546772 |
| }, |
| "truthfulqa_mc2": { |
| "alias": "truthfulqa_mc2", |
| "acc,none": 0.5284457708979753, |
| "acc_stderr,none": 0.015035693028473887 |
| } |
| }, |
| "Mistral-Nemo-Instruct-2407-NVFP4-4over6": { |
| "coqa": { |
| "alias": "coqa", |
| "em,none": 0.5498333333333334, |
| "em_stderr,none": 0.020187689512123422, |
| "f1,none": 0.7211863549140475, |
| "f1_stderr,none": 0.01540089217977273 |
| }, |
| "hellaswag": { |
| "alias": "hellaswag", |
| "acc,none": 0.619398526190002, |
| "acc_stderr,none": 0.004845424524763779, |
| "acc_norm,none": 0.8131846245767775, |
| "acc_norm_stderr,none": 0.003889666837869389 |
| }, |
| "ifeval": { |
| "alias": "ifeval", |
| "prompt_level_strict_acc,none": 0.4121996303142329, |
| "prompt_level_strict_acc_stderr,none": 0.021182238151733295, |
| "inst_level_strict_acc,none": 0.5095923261390888, |
| "inst_level_strict_acc_stderr,none": "N/A", |
| "prompt_level_loose_acc,none": 0.4824399260628466, |
| "prompt_level_loose_acc_stderr,none": 0.02150330051338897, |
| "inst_level_loose_acc,none": 0.5683453237410072, |
| "inst_level_loose_acc_stderr,none": "N/A" |
| }, |
| "lambada_openai": { |
| "alias": "lambada_openai", |
| "perplexity,none": 2.954572513479153, |
| "perplexity_stderr,none": 0.054124688702298446, |
| "acc,none": 0.7686784397438385, |
| "acc_stderr,none": 0.0058747917899013785 |
| }, |
| "lambada_openai_cloze_yaml": { |
| "alias": "lambada_openai_cloze_yaml", |
| "perplexity,none": 30.035509240670926, |
| "perplexity_stderr,none": 0.7780223123588234, |
| "acc,none": 0.29827285076654375, |
| "acc_stderr,none": 0.006373868144287074 |
| }, |
| "lambada_standard": { |
| "alias": "lambada_standard", |
| "perplexity,none": 3.660039154584711, |
| "perplexity_stderr,none": 0.07563902292271452, |
| "acc,none": 0.6906656316708714, |
| "acc_stderr,none": 0.006439617662597691 |
| }, |
| "lambada_standard_cloze_yaml": { |
| "alias": "lambada_standard_cloze_yaml", |
| "perplexity,none": 40.99251979932923, |
| "perplexity_stderr,none": 1.0271212425445286, |
| "acc,none": 0.24665243547448087, |
| "acc_stderr,none": 0.006005545631215194 |
| }, |
| "commonsense_qa": { |
| "alias": "commonsense_qa", |
| "acc,none": 0.5921375921375921, |
| "acc_stderr,none": 0.014069810259917194 |
| }, |
| "mmlu": { |
| "acc,none": 0.6364477994587665, |
| "acc_stderr,none": 0.003806377839571922, |
| "alias": "mmlu" |
| }, |
| "openbookqa": { |
| "alias": "openbookqa", |
| "acc,none": 0.392, |
| "acc_stderr,none": 0.02185468495561119, |
| "acc_norm,none": 0.472, |
| "acc_norm_stderr,none": 0.022347949832668024 |
| }, |
| "winogrande": { |
| "alias": "winogrande", |
| "acc,none": 0.755327545382794, |
| "acc_stderr,none": 0.012082125654159727 |
| }, |
| "triviaqa": { |
| "alias": "triviaqa", |
| "exact_match,remove_whitespace": 0.6011480160499332, |
| "exact_match_stderr,remove_whitespace": 0.003655519111850352 |
| }, |
| "truthfulqa_mc1": { |
| "alias": "truthfulqa_mc1", |
| "acc,none": 0.38310893512851896, |
| "acc_stderr,none": 0.017018461679389734 |
| }, |
| "truthfulqa_mc2": { |
| "alias": "truthfulqa_mc2", |
| "acc,none": 0.5367421440427503, |
| "acc_stderr,none": 0.01489354828588867 |
| } |
| }, |
| "Mistral-Nemo-Instruct-2407-NVFP4-FP8-RTN": { |
| "coqa": { |
| "alias": "coqa", |
| "em,none": 0.5683333333333334, |
| "em_stderr,none": 0.019596946262820592, |
| "f1,none": 0.7401341567024432, |
| "f1_stderr,none": 0.014222135053403443 |
| }, |
| "hellaswag": { |
| "alias": "hellaswag", |
| "acc,none": 0.6237801234813782, |
| "acc_stderr,none": 0.004834461997944986, |
| "acc_norm,none": 0.813981278629755, |
| "acc_norm_stderr,none": 0.003883265210791469 |
| }, |
| "ifeval": { |
| "alias": "ifeval", |
| "prompt_level_strict_acc,none": 0.39926062846580407, |
| "prompt_level_strict_acc_stderr,none": 0.021075331332701258, |
| "inst_level_strict_acc,none": 0.5011990407673861, |
| "inst_level_strict_acc_stderr,none": "N/A", |
| "prompt_level_loose_acc,none": 0.46210720887245843, |
| "prompt_level_loose_acc_stderr,none": 0.021454695436204742, |
| "inst_level_loose_acc,none": 0.5563549160671463, |
| "inst_level_loose_acc_stderr,none": "N/A" |
| }, |
| "lambada_openai": { |
| "alias": "lambada_openai", |
| "perplexity,none": 2.959118986596569, |
| "perplexity_stderr,none": 0.05562968630849542, |
| "acc,none": 0.7618862798369882, |
| "acc_stderr,none": 0.005934024831865026 |
| }, |
| "lambada_openai_cloze_yaml": { |
| "alias": "lambada_openai_cloze_yaml", |
| "perplexity,none": 26.696978874621955, |
| "perplexity_stderr,none": 0.683775381967173, |
| "acc,none": 0.33145740345429847, |
| "acc_stderr,none": 0.006558287884402134 |
| }, |
| "lambada_standard": { |
| "alias": "lambada_standard", |
| "perplexity,none": 3.492972951792885, |
| "perplexity_stderr,none": 0.0721179626815854, |
| "acc,none": 0.6970696681544731, |
| "acc_stderr,none": 0.006402086620816973 |
| }, |
| "lambada_standard_cloze_yaml": { |
| "alias": "lambada_standard_cloze_yaml", |
| "perplexity,none": 37.411029419339414, |
| "perplexity_stderr,none": 0.9371152123007664, |
| "acc,none": 0.2582961381719387, |
| "acc_stderr,none": 0.0060979842659205745 |
| }, |
| "commonsense_qa": { |
| "alias": "commonsense_qa", |
| "acc,none": 0.6060606060606061, |
| "acc_stderr,none": 0.013989198052984327 |
| }, |
| "mmlu": { |
| "acc,none": 0.6434268622703319, |
| "acc_stderr,none": 0.0037824170513249015, |
| "alias": "mmlu" |
| }, |
| "openbookqa": { |
| "alias": "openbookqa", |
| "acc,none": 0.404, |
| "acc_stderr,none": 0.021966635293832883, |
| "acc_norm,none": 0.478, |
| "acc_norm_stderr,none": 0.02236139673920787 |
| }, |
| "winogrande": { |
| "alias": "winogrande", |
| "acc,none": 0.7513812154696132, |
| "acc_stderr,none": 0.012147314713403173 |
| }, |
| "triviaqa": { |
| "alias": "triviaqa", |
| "exact_match,remove_whitespace": 0.6104547481052163, |
| "exact_match_stderr,remove_whitespace": 0.0036404759558834486 |
| }, |
| "truthfulqa_mc1": { |
| "alias": "truthfulqa_mc1", |
| "acc,none": 0.38922888616891066, |
| "acc_stderr,none": 0.01706855268069044 |
| }, |
| "truthfulqa_mc2": { |
| "alias": "truthfulqa_mc2", |
| "acc,none": 0.5389966378633866, |
| "acc_stderr,none": 0.015062753562771725 |
| } |
| }, |
| "Mistral-Nemo-Instruct-2407-NVFP4-FP8": { |
| "coqa": { |
| "alias": "coqa", |
| "em,none": 0.5733333333333334, |
| "em_stderr,none": 0.019505122108457063, |
| "f1,none": 0.7346963191068078, |
| "f1_stderr,none": 0.015019925951016882 |
| }, |
| "hellaswag": { |
| "alias": "hellaswag", |
| "acc,none": 0.6239792869946226, |
| "acc_stderr,none": 0.004833953712521647, |
| "acc_norm,none": 0.8124875522804222, |
| "acc_norm_stderr,none": 0.0038952463204528333 |
| }, |
| "ifeval": { |
| "alias": "ifeval", |
| "prompt_level_strict_acc,none": 0.4195933456561922, |
| "prompt_level_strict_acc_stderr,none": 0.021236532548855144, |
| "inst_level_strict_acc,none": 0.5107913669064749, |
| "inst_level_strict_acc_stderr,none": "N/A", |
| "prompt_level_loose_acc,none": 0.49168207024029575, |
| "prompt_level_loose_acc_stderr,none": 0.02151359656402127, |
| "inst_level_loose_acc,none": 0.5767386091127098, |
| "inst_level_loose_acc_stderr,none": "N/A" |
| }, |
| "lambada_openai": { |
| "alias": "lambada_openai", |
| "perplexity,none": 2.923326330881571, |
| "perplexity_stderr,none": 0.05419037402311029, |
| "acc,none": 0.7725596739763245, |
| "acc_stderr,none": 0.005839986255519642 |
| }, |
| "lambada_openai_cloze_yaml": { |
| "alias": "lambada_openai_cloze_yaml", |
| "perplexity,none": 26.694758918393248, |
| "perplexity_stderr,none": 0.6858369120887257, |
| "acc,none": 0.33165146516592275, |
| "acc_stderr,none": 0.0065592552732574244 |
| }, |
| "lambada_standard": { |
| "alias": "lambada_standard", |
| "perplexity,none": 3.551356285449324, |
| "perplexity_stderr,none": 0.07342709527808654, |
| "acc,none": 0.6926062487871143, |
| "acc_stderr,none": 0.006428398527904964 |
| }, |
| "lambada_standard_cloze_yaml": { |
| "alias": "lambada_standard_cloze_yaml", |
| "perplexity,none": 35.56146784383732, |
| "perplexity_stderr,none": 0.8740618781960072, |
| "acc,none": 0.2837182223947215, |
| "acc_stderr,none": 0.0062805494483705775 |
| }, |
| "commonsense_qa": { |
| "alias": "commonsense_qa", |
| "acc,none": 0.6208026208026208, |
| "acc_stderr,none": 0.013890872306969766 |
| }, |
| "mmlu": { |
| "acc,none": 0.6454208802164934, |
| "acc_stderr,none": 0.0037774970297410814, |
| "alias": "mmlu" |
| }, |
| "openbookqa": { |
| "alias": "openbookqa", |
| "acc,none": 0.404, |
| "acc_stderr,none": 0.021966635293832883, |
| "acc_norm,none": 0.488, |
| "acc_norm_stderr,none": 0.022376626797927058 |
| }, |
| "winogrande": { |
| "alias": "winogrande", |
| "acc,none": 0.7545382794001578, |
| "acc_stderr,none": 0.01209527293718361 |
| }, |
| "triviaqa": { |
| "alias": "triviaqa", |
| "exact_match,remove_whitespace": 0.6184239857333927, |
| "exact_match_stderr,remove_whitespace": 0.003626487357735664 |
| }, |
| "truthfulqa_mc1": { |
| "alias": "truthfulqa_mc1", |
| "acc,none": 0.39167686658506734, |
| "acc_stderr,none": 0.017087795881769625 |
| }, |
| "truthfulqa_mc2": { |
| "alias": "truthfulqa_mc2", |
| "acc,none": 0.5475484445657804, |
| "acc_stderr,none": 0.015041357055984873 |
| } |
| } |
| } |
