{ "Qwen3.8-27B-w4a16-vision-run1": { "started": "2026-08-30 06:17:05", "finished": "2026-08-30 07:57:21", "seed": 20260816, "thinking": true, "tasks": { "mmlu": { "task": "mmlu", "n": 150, "hits": 130, "accuracy_pct": 86.67, "accuracy_ex_trunc_pct": 90.28, "truncated": 6, "completion_tokens_p50": 265, "completion_tokens_p90": 6062, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 747.3, "gen_tokens": 416314, "tok_per_s": 557.1, "details": [ { "id": "mmlu-12325", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-154", "ok": true, "error": null, "completion_tokens": 111, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12844", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-1153", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11089", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12782", "ok": true, "error": null, "completion_tokens": 867, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5422", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12363", "ok": true, "error": null, "completion_tokens": 8756, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12556", "ok": true, "error": null, "completion_tokens": 338, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13163", "ok": true, "error": null, "completion_tokens": 2004, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-270", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6943", "ok": true, "error": null, "completion_tokens": 659, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8330", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9276", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7584", "ok": true, "error": null, "completion_tokens": 51, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12819", "ok": true, "error": null, "completion_tokens": 476, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6163", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4099", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10389", "ok": false, "error": null, "completion_tokens": 6062, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1539", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12013", "ok": true, "error": null, "completion_tokens": 18388, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7814", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1106", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1616", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-555", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3698", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-618", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9842", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4229", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1022", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6421", "ok": true, "error": null, "completion_tokens": 724, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1997", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8941", "ok": false, "error": null, "completion_tokens": 620, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-565", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11364", "ok": true, "error": null, "completion_tokens": 5415, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5959", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13066", "ok": false, "error": null, "completion_tokens": 3965, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3122", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3722", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13839", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12570", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9959", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10791", "ok": true, "error": null, "completion_tokens": 15151, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12268", "ok": true, "error": null, "completion_tokens": 772, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9051", "ok": true, "error": null, "completion_tokens": 652, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12887", "ok": true, "error": null, "completion_tokens": 789, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2667", "ok": true, "error": null, "completion_tokens": 5927, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3906", "ok": true, "error": null, "completion_tokens": 413, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10304", "ok": true, "error": null, "completion_tokens": 51, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12927", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6337", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3907", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12888", "ok": false, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10629", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5801", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2936", "ok": true, "error": null, "completion_tokens": 254, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7389", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9532", "ok": false, "error": null, "completion_tokens": 2492, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8705", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11776", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-6836", "ok": true, "error": null, "completion_tokens": 3171, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7856", "ok": true, "error": null, "completion_tokens": 1187, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9697", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10825", "ok": true, "error": null, "completion_tokens": 7241, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11178", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-11420", "ok": true, "error": null, "completion_tokens": 3772, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4658", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5403", "ok": false, "error": null, "completion_tokens": 1864, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6160", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6872", "ok": false, "error": null, "completion_tokens": 15877, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9738", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3076", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13496", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11289", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-222", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12180", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1143", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13086", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7038", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13587", "ok": false, "error": null, "completion_tokens": 260, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12349", "ok": true, "error": null, "completion_tokens": 320, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11747", "ok": true, "error": null, "completion_tokens": 269, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9848", "ok": false, "error": null, "completion_tokens": 11430, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3438", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-89", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4253", "ok": true, "error": null, "completion_tokens": 116, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5460", "ok": true, "error": null, "completion_tokens": 1308, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2494", "ok": false, "error": null, "completion_tokens": 15659, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11149", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-718", "ok": true, "error": null, "completion_tokens": 1159, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5330", "ok": false, "error": null, "completion_tokens": 3801, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10716", "ok": true, "error": null, "completion_tokens": 870, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5636", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11832", "ok": true, "error": null, "completion_tokens": 1167, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-95", "ok": true, "error": null, "completion_tokens": 515, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13960", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2072", "ok": true, "error": null, "completion_tokens": 76, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12030", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-2931", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2441", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13666", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6575", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11625", "ok": true, "error": null, "completion_tokens": 926, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13283", "ok": false, "error": null, "completion_tokens": 11641, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4262", "ok": true, "error": null, "completion_tokens": 325, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12931", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12150", "ok": true, "error": null, "completion_tokens": 6002, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3678", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4593", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1563", "ok": true, "error": null, "completion_tokens": 2862, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13396", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8049", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8396", "ok": true, "error": null, "completion_tokens": 606, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8493", "ok": true, "error": null, "completion_tokens": 662, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13514", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3000", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1635", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1391", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-627", "ok": false, "error": null, "completion_tokens": 17817, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1573", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6731", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11665", "ok": true, "error": null, "completion_tokens": 3955, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2914", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6875", "ok": true, "error": null, "completion_tokens": 3859, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2184", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12801", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6717", "ok": true, "error": null, "completion_tokens": 542, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12464", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2787", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11549", "ok": true, "error": null, "completion_tokens": 487, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-842", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13868", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9647", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9502", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12441", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4392", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5036", "ok": true, "error": null, "completion_tokens": 740, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12606", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1400", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13979", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13634", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11226", "ok": true, "error": null, "completion_tokens": 4747, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-625", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-257", "ok": true, "error": null, "completion_tokens": 555, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12853", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-451", "ok": true, "error": null, "completion_tokens": 376, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2074", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9117", "ok": false, "error": null, "completion_tokens": 2752, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10748", "ok": true, "error": null, "completion_tokens": 364, "reasoning_chars": 0, "truncated": false } ] }, "cmmlu": { "task": "cmmlu", "n": 150, "hits": 135, "accuracy_pct": 90.0, "accuracy_ex_trunc_pct": 91.22, "truncated": 2, "completion_tokens_p50": 338, "completion_tokens_p90": 2729, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 573.1, "gen_tokens": 213946, "tok_per_s": 373.3, "details": [ { "id": "cmmlu-chinese_teacher_qualification-63", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-modern_chinese-54", "ok": true, "error": null, "completion_tokens": 5084, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-91", "ok": true, "error": null, "completion_tokens": 543, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-260", "ok": false, "error": null, "completion_tokens": 615, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-32", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marxist_theory-29", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_history-200", "ok": true, "error": null, "completion_tokens": 5133, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-98", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-44", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_politics-118", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-116", "ok": true, "error": null, "completion_tokens": 3419, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-43", "ok": true, "error": null, "completion_tokens": 282, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-145", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-247", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sociology-224", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_mathematics-26", "ok": true, "error": null, "completion_tokens": 882, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-105", "ok": true, "error": null, "completion_tokens": 1547, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-logical-15", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medicine-62", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-nutrition-74", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_literature-181", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-127", "ok": true, "error": null, "completion_tokens": 727, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_geography-107", "ok": true, "error": null, "completion_tokens": 841, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-agronomy-18", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-82", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_food_culture-38", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-198", "ok": true, "error": null, "completion_tokens": 385, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-96", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-73", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "cmmlu-genetics-20", "ok": true, "error": null, "completion_tokens": 273, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-logical-95", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-88", "ok": true, "error": null, "completion_tokens": 763, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-business_ethics-192", "ok": true, "error": null, "completion_tokens": 639, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marxist_theory-66", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-100", "ok": true, "error": null, "completion_tokens": 320, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-136", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-16", "ok": false, "error": null, "completion_tokens": 394, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-logical-90", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sports_science-41", "ok": true, "error": null, "completion_tokens": 3287, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-345", "ok": true, "error": null, "completion_tokens": 457, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sociology-103", "ok": false, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-agronomy-93", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-88", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-206", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-105", "ok": true, "error": null, "completion_tokens": 802, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-economics-90", "ok": true, "error": null, "completion_tokens": 86, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-190", "ok": false, "error": null, "completion_tokens": 304, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-international_law-39", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-155", "ok": true, "error": null, "completion_tokens": 1160, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medicine-24", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-6", "ok": true, "error": null, "completion_tokens": 422, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_mathematics-31", "ok": true, "error": null, "completion_tokens": 574, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-243", "ok": true, "error": null, "completion_tokens": 665, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_accounting-63", "ok": true, "error": null, "completion_tokens": 3117, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sports_science-135", "ok": true, "error": null, "completion_tokens": 601, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-89", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-91", "ok": false, "error": null, "completion_tokens": 2729, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_psychology-120", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-62", "ok": true, "error": null, "completion_tokens": 1105, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-106", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-food_science-106", "ok": true, "error": null, "completion_tokens": 14995, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-24", "ok": true, "error": null, "completion_tokens": 1793, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-157", "ok": true, "error": null, "completion_tokens": 584, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-41", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-115", "ok": true, "error": null, "completion_tokens": 6875, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-journalism-29", "ok": false, "error": null, "completion_tokens": 4632, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-67", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-161", "ok": true, "error": null, "completion_tokens": 733, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-virology-120", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-education-117", "ok": true, "error": null, "completion_tokens": 84, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-169", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-conceptual_physics-129", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-machine_learning-19", "ok": true, "error": null, "completion_tokens": 1766, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-8", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-72", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-public_relations-139", "ok": false, "error": null, "completion_tokens": 862, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_security-22", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-7", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-108", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-31", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-131", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-35", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "cmmlu-college_actuarial_science-38", "ok": true, "error": null, "completion_tokens": 1566, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-165", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-101", "ok": true, "error": null, "completion_tokens": 2266, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-107", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-machine_learning-8", "ok": true, "error": null, "completion_tokens": 6232, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medical_statistics-86", "ok": true, "error": null, "completion_tokens": 338, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-security_study-30", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-118", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_law-24", "ok": false, "error": null, "completion_tokens": 3082, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_history-46", "ok": true, "error": null, "completion_tokens": 551, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-anatomy-21", "ok": false, "error": null, "completion_tokens": 8803, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-145", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-arts-91", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-109", "ok": false, "error": null, "completion_tokens": 155, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-machine_learning-44", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-74", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-63", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-astronomy-146", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-business_ethics-109", "ok": true, "error": null, "completion_tokens": 418, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-construction_project_management-16", "ok": true, "error": null, "completion_tokens": 288, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marxist_theory-113", "ok": false, "error": null, "completion_tokens": 475, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-102", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-207", "ok": true, "error": null, "completion_tokens": 343, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-economics-19", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-164", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-338", "ok": true, "error": null, "completion_tokens": 909, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_politics-7", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-virology-92", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-47", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-astronomy-140", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-100", "ok": true, "error": null, "completion_tokens": 758, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-37", "ok": true, "error": null, "completion_tokens": 992, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-191", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medicine-34", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_accounting-88", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-public_relations-77", "ok": false, "error": null, "completion_tokens": 306, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_history-306", "ok": true, "error": null, "completion_tokens": 7821, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-business_ethics-40", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_chemistry-7", "ok": true, "error": null, "completion_tokens": 1899, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-152", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-1", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-94", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_psychology-109", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_literature-164", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sports_science-89", "ok": false, "error": null, "completion_tokens": 1828, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-economics-17", "ok": true, "error": null, "completion_tokens": 517, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-arts-2", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-51", "ok": true, "error": null, "completion_tokens": 574, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-agronomy-141", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-196", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-97", "ok": true, "error": null, "completion_tokens": 712, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-202", "ok": true, "error": null, "completion_tokens": 12281, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-195", "ok": true, "error": null, "completion_tokens": 1005, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-conceptual_physics-21", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sociology-49", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-94", "ok": true, "error": null, "completion_tokens": 1010, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-8", "ok": true, "error": null, "completion_tokens": 780, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-98", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-modern_chinese-28", "ok": true, "error": null, "completion_tokens": 269, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-74", "ok": true, "error": null, "completion_tokens": 1097, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-121", "ok": true, "error": null, "completion_tokens": 528, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_mathematics-120", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-105", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-35", "ok": true, "error": null, "completion_tokens": 1590, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_accounting-51", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_mathematics-49", "ok": true, "error": null, "completion_tokens": 2438, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_politics-130", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-virology-140", "ok": true, "error": null, "completion_tokens": 807, "reasoning_chars": 0, "truncated": false } ] }, "mmlu_pro": { "task": "mmlu_pro", "n": 150, "hits": 132, "accuracy_pct": 88.0, "accuracy_ex_trunc_pct": 91.03, "truncated": 5, "completion_tokens_p50": 462, "completion_tokens_p90": 5861, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 825.6, "gen_tokens": 410971, "tok_per_s": 497.8, "details": [ { "id": "mmlupro-5831", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2299", "ok": true, "error": null, "completion_tokens": 728, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9280", "ok": true, "error": null, "completion_tokens": 5070, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5486", "ok": true, "error": null, "completion_tokens": 1492, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5761", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6589", "ok": true, "error": null, "completion_tokens": 831, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5389", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5131", "ok": true, "error": null, "completion_tokens": 767, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1677", "ok": true, "error": null, "completion_tokens": 3376, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4388", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-3285", "ok": true, "error": null, "completion_tokens": 1409, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4118", "ok": true, "error": null, "completion_tokens": 2076, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5344", "ok": true, "error": null, "completion_tokens": 609, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11696", "ok": true, "error": null, "completion_tokens": 1716, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-858", "ok": true, "error": null, "completion_tokens": 701, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-214", "ok": true, "error": null, "completion_tokens": 6821, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6609", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2083", "ok": true, "error": null, "completion_tokens": 578, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9752", "ok": false, "error": null, "completion_tokens": 1367, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7213", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5234", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8570", "ok": true, "error": null, "completion_tokens": 137, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5612", "ok": true, "error": null, "completion_tokens": 889, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2071", "ok": true, "error": null, "completion_tokens": 387, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9842", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4665", "ok": true, "error": null, "completion_tokens": 907, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5564", "ok": true, "error": null, "completion_tokens": 2369, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6475", "ok": true, "error": null, "completion_tokens": 1636, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2153", "ok": true, "error": null, "completion_tokens": 566, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5812", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3407", "ok": true, "error": null, "completion_tokens": 5562, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2879", "ok": false, "error": null, "completion_tokens": 2146, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8266", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1916", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-3486", "ok": true, "error": null, "completion_tokens": 2054, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11275", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6040", "ok": true, "error": null, "completion_tokens": 474, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8565", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6848", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8926", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8458", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7854", "ok": true, "error": null, "completion_tokens": 531, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3962", "ok": true, "error": null, "completion_tokens": 1291, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6470", "ok": true, "error": null, "completion_tokens": 500, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3813", "ok": true, "error": null, "completion_tokens": 421, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8026", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11030", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8151", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5286", "ok": true, "error": null, "completion_tokens": 560, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3888", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11676", "ok": true, "error": null, "completion_tokens": 2095, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8762", "ok": false, "error": null, "completion_tokens": 1202, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-12160", "ok": true, "error": null, "completion_tokens": 529, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4022", "ok": true, "error": null, "completion_tokens": 790, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10092", "ok": true, "error": null, "completion_tokens": 720, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7364", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6001", "ok": true, "error": null, "completion_tokens": 453, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10873", "ok": true, "error": null, "completion_tokens": 447, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9777", "ok": true, "error": null, "completion_tokens": 7222, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8723", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4595", "ok": true, "error": null, "completion_tokens": 546, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8423", "ok": true, "error": null, "completion_tokens": 8068, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11051", "ok": false, "error": null, "completion_tokens": 137, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-836", "ok": true, "error": null, "completion_tokens": 363, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11297", "ok": true, "error": null, "completion_tokens": 29048, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3950", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1562", "ok": false, "error": null, "completion_tokens": 4530, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2041", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3473", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11253", "ok": true, "error": null, "completion_tokens": 1138, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3858", "ok": true, "error": null, "completion_tokens": 872, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2231", "ok": true, "error": null, "completion_tokens": 462, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2190", "ok": true, "error": null, "completion_tokens": 612, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6930", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6558", "ok": true, "error": null, "completion_tokens": 275, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5371", "ok": true, "error": null, "completion_tokens": 11358, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1766", "ok": true, "error": null, "completion_tokens": 1161, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6045", "ok": true, "error": null, "completion_tokens": 4335, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2298", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6400", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6499", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7936", "ok": true, "error": null, "completion_tokens": 459, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8566", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11633", "ok": true, "error": null, "completion_tokens": 418, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9843", "ok": false, "error": null, "completion_tokens": 4270, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11758", "ok": false, "error": null, "completion_tokens": 12698, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2843", "ok": true, "error": null, "completion_tokens": 2722, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7483", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11379", "ok": true, "error": null, "completion_tokens": 873, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4866", "ok": true, "error": null, "completion_tokens": 494, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-818", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2785", "ok": true, "error": null, "completion_tokens": 1781, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8119", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2214", "ok": true, "error": null, "completion_tokens": 2584, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5774", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-6049", "ok": false, "error": null, "completion_tokens": 2962, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7219", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1660", "ok": false, "error": null, "completion_tokens": 448, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4767", "ok": false, "error": null, "completion_tokens": 14713, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5964", "ok": false, "error": null, "completion_tokens": 10492, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2743", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4062", "ok": true, "error": null, "completion_tokens": 403, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7368", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-827", "ok": false, "error": null, "completion_tokens": 527, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11877", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-7557", "ok": true, "error": null, "completion_tokens": 425, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9998", "ok": true, "error": null, "completion_tokens": 373, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10098", "ok": false, "error": null, "completion_tokens": 9722, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1744", "ok": true, "error": null, "completion_tokens": 3910, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9210", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11248", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11187", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2799", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4479", "ok": true, "error": null, "completion_tokens": 1293, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10223", "ok": true, "error": null, "completion_tokens": 1069, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5698", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8291", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3516", "ok": true, "error": null, "completion_tokens": 277, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5466", "ok": true, "error": null, "completion_tokens": 577, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4851", "ok": true, "error": null, "completion_tokens": 553, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-733", "ok": true, "error": null, "completion_tokens": 3847, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6995", "ok": true, "error": null, "completion_tokens": 332, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2573", "ok": true, "error": null, "completion_tokens": 1150, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8481", "ok": true, "error": null, "completion_tokens": 208, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5938", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10708", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8476", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7990", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2409", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11551", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-674", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1978", "ok": true, "error": null, "completion_tokens": 8654, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10420", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3445", "ok": true, "error": null, "completion_tokens": 4583, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9488", "ok": true, "error": null, "completion_tokens": 343, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4993", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4316", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4730", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11570", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2149", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8322", "ok": true, "error": null, "completion_tokens": 826, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10993", "ok": true, "error": null, "completion_tokens": 5098, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4588", "ok": true, "error": null, "completion_tokens": 5861, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3258", "ok": true, "error": null, "completion_tokens": 591, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8629", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-637", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4257", "ok": true, "error": null, "completion_tokens": 460, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9719", "ok": true, "error": null, "completion_tokens": 805, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11714", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-9678", "ok": true, "error": null, "completion_tokens": 5106, "reasoning_chars": 0, "truncated": false } ] }, "ceval": { "task": "ceval", "n": 150, "hits": 127, "accuracy_pct": 84.67, "accuracy_ex_trunc_pct": 85.81, "truncated": 2, "completion_tokens_p50": 435, "completion_tokens_p90": 5504, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 686.8, "gen_tokens": 307905, "tok_per_s": 448.3, "details": [ { "id": "ceval-high_school_chinese-1", "ok": false, "error": null, "completion_tokens": 7078, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-10", "ok": true, "error": null, "completion_tokens": 709, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-41", "ok": false, "error": null, "completion_tokens": 16733, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_physics-3", "ok": true, "error": null, "completion_tokens": 252, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-21", "ok": false, "error": null, "completion_tokens": 987, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-advanced_mathematics-4", "ok": true, "error": null, "completion_tokens": 679, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_history-21", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-2", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-6", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-12", "ok": true, "error": null, "completion_tokens": 12366, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-19", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_chemistry-9", "ok": true, "error": null, "completion_tokens": 2040, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-15", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-22", "ok": false, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-16", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-11", "ok": true, "error": null, "completion_tokens": 702, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-18", "ok": true, "error": null, "completion_tokens": 224, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-24", "ok": true, "error": null, "completion_tokens": 527, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-29", "ok": true, "error": null, "completion_tokens": 1631, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_geography-2", "ok": true, "error": null, "completion_tokens": 782, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-11", "ok": false, "error": null, "completion_tokens": 2037, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-18", "ok": true, "error": null, "completion_tokens": 611, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-0", "ok": false, "error": null, "completion_tokens": 1185, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-veterinary_medicine-14", "ok": true, "error": null, "completion_tokens": 325, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-27", "ok": true, "error": null, "completion_tokens": 1586, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-art_studies-19", "ok": true, "error": null, "completion_tokens": 2595, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-21", "ok": true, "error": null, "completion_tokens": 5504, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-33", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-veterinary_medicine-20", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-10", "ok": true, "error": null, "completion_tokens": 565, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-19", "ok": true, "error": null, "completion_tokens": 111, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-computer_network-7", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-41", "ok": true, "error": null, "completion_tokens": 509, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-urban_and_rural_planner-44", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-9", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-28", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-39", "ok": true, "error": null, "completion_tokens": 1411, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-1", "ok": false, "error": null, "completion_tokens": 1302, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-24", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-5", "ok": true, "error": null, "completion_tokens": 7404, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-computer_architecture-15", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-39", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-law-23", "ok": true, "error": null, "completion_tokens": 502, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-clinical_medicine-7", "ok": true, "error": null, "completion_tokens": 703, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-17", "ok": true, "error": null, "completion_tokens": 1526, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_chemistry-6", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-15", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_mathematics-4", "ok": false, "error": null, "completion_tokens": 17719, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-5", "ok": true, "error": null, "completion_tokens": 1080, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-50", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_politics-11", "ok": true, "error": null, "completion_tokens": 507, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-32", "ok": true, "error": null, "completion_tokens": 2394, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-16", "ok": true, "error": null, "completion_tokens": 456, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-0", "ok": true, "error": null, "completion_tokens": 711, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-sports_science-9", "ok": true, "error": null, "completion_tokens": 335, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-art_studies-15", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-2", "ok": false, "error": null, "completion_tokens": 7647, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-7", "ok": false, "error": null, "completion_tokens": 3154, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-11", "ok": true, "error": null, "completion_tokens": 702, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-operating_system-18", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_physics-4", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-17", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-1", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-32", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-33", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-education_science-25", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-40", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-7", "ok": true, "error": null, "completion_tokens": 5125, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-modern_chinese_history-5", "ok": true, "error": null, "completion_tokens": 1394, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-12", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-modern_chinese_history-4", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-38", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "ceval-electrical_engineer-22", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-clinical_medicine-0", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-15", "ok": false, "error": null, "completion_tokens": 3296, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-45", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-6", "ok": true, "error": null, "completion_tokens": 416, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-20", "ok": true, "error": null, "completion_tokens": 3290, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-basic_medicine-2", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-17", "ok": true, "error": null, "completion_tokens": 435, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-16", "ok": false, "error": null, "completion_tokens": 622, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-7", "ok": true, "error": null, "completion_tokens": 537, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_biology-16", "ok": false, "error": null, "completion_tokens": 238, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-10", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-32", "ok": true, "error": null, "completion_tokens": 4837, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_geography-0", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-12", "ok": false, "error": null, "completion_tokens": 5182, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-14", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-26", "ok": false, "error": null, "completion_tokens": 12536, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-3", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-professional_tour_guide-3", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-computer_network-13", "ok": true, "error": null, "completion_tokens": 64, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-sports_science-5", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-27", "ok": true, "error": null, "completion_tokens": 911, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-21", "ok": true, "error": null, "completion_tokens": 11287, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-urban_and_rural_planner-10", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-16", "ok": true, "error": null, "completion_tokens": 10215, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-49", "ok": true, "error": null, "completion_tokens": 611, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-13", "ok": false, "error": null, "completion_tokens": 3619, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-5", "ok": true, "error": null, "completion_tokens": 9942, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-2", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-27", "ok": false, "error": null, "completion_tokens": 5582, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_biology-8", "ok": true, "error": null, "completion_tokens": 613, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_physics-14", "ok": true, "error": null, "completion_tokens": 878, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_history-14", "ok": true, "error": null, "completion_tokens": 205, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-31", "ok": true, "error": null, "completion_tokens": 597, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-basic_medicine-13", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_chemistry-13", "ok": true, "error": null, "completion_tokens": 5337, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-12", "ok": true, "error": null, "completion_tokens": 542, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-44", "ok": false, "error": null, "completion_tokens": 8661, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_mathematics-3", "ok": true, "error": null, "completion_tokens": 1745, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-13", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "ceval-middle_school_mathematics-6", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_politics-19", "ok": true, "error": null, "completion_tokens": 602, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-environmental_impact_assessment_engineer-27", "ok": false, "error": null, "completion_tokens": 4049, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-law-10", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-law-0", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-3", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-modern_chinese_history-6", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_chinese-13", "ok": true, "error": null, "completion_tokens": 638, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-urban_and_rural_planner-43", "ok": false, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-42", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_mathematics-13", "ok": true, "error": null, "completion_tokens": 580, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-discrete_mathematics-1", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-11", "ok": true, "error": null, "completion_tokens": 1181, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_history-16", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-2", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-1", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_physics-16", "ok": true, "error": null, "completion_tokens": 6205, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_chemistry-11", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-5", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-40", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_physics-6", "ok": true, "error": null, "completion_tokens": 347, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_mathematics-10", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-10", "ok": true, "error": null, "completion_tokens": 1658, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-education_science-17", "ok": true, "error": null, "completion_tokens": 1735, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-11", "ok": true, "error": null, "completion_tokens": 614, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-professional_tour_guide-16", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-education_science-1", "ok": false, "error": null, "completion_tokens": 2888, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_physics-0", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-37", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-9", "ok": true, "error": null, "completion_tokens": 84, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_history-16", "ok": true, "error": null, "completion_tokens": 823, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-5", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-advanced_mathematics-18", "ok": true, "error": null, "completion_tokens": 465, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-30", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-21", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-42", "ok": true, "error": null, "completion_tokens": 569, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-veterinary_medicine-5", "ok": true, "error": null, "completion_tokens": 1371, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_chemistry-4", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 0, "truncated": false } ] }, "arc": { "task": "arc", "n": 150, "hits": 144, "accuracy_pct": 96.0, "accuracy_ex_trunc_pct": 96.0, "truncated": 0, "completion_tokens_p50": 135, "completion_tokens_p90": 323, "completion_tokens_max": 7550, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 127.2, "gen_tokens": 45568, "tok_per_s": 358.3, "details": [ { "id": "arc-Mercury_415084", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7249708", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_4_24", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7056823", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_415534", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_405827", "ok": false, "error": null, "completion_tokens": 7550, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7263183", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7106960", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2003_8_7", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7182245", "ok": true, "error": null, "completion_tokens": 2752, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416177", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 0, "truncated": false }, { "id": "arc-ACTAAP_2011_5_8", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TAKS_2009_8_36", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_401699", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7107240", "ok": true, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AKDE&ED_2008_4_26", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402638", "ok": true, "error": null, "completion_tokens": 671, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7026723", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_415422", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP_2000_8_2", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2006_9_4", "ok": true, "error": null, "completion_tokens": 284, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401272", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416169", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7116235", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7109463", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7032428", "ok": true, "error": null, "completion_tokens": 7156, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AIMS_2008_8_11", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_410334", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TAKS_2009_5_36", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_415719", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2005_9_17", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402256", "ok": true, "error": null, "completion_tokens": 205, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_400443", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7133245", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_413458", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401372", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401116", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP__5_10312", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP_2007_4_10280", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_416683", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7190120", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401295", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_4_28", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7223493", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7018410", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_1998_8_24", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2003_8_31", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_405501", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_408030", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_401684", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7041668", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7205870", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7210140", "ok": true, "error": null, "completion_tokens": 275, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_405873", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400052", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7018270", "ok": true, "error": null, "completion_tokens": 370, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_406550", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7150728", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400839", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false }, { "id": "arc-VASoL_2009_5_30", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400383", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7236740", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 0, "truncated": false }, { "id": "arc-CSZ_2005_5_CSZ10021", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7093958", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7026355", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_408324", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP__7_10348", "ok": true, "error": null, "completion_tokens": 84, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402629", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400046", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7180810", "ok": true, "error": null, "completion_tokens": 337, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_180005", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AIMS_2009_8_14", "ok": true, "error": null, "completion_tokens": 328, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2007_5_16", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_LBS10662", "ok": true, "error": null, "completion_tokens": 124, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2013_8_23", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2012_8_19", "ok": false, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7041650", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416181", "ok": true, "error": null, "completion_tokens": 230, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TIMSS_1995_8_J6", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2014_5_13", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7106750", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_1999_4_23", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MEA_2013_5_12", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_8_42", "ok": false, "error": null, "completion_tokens": 431, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_408367", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7250058", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_414041", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401290", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_409295", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7211680", "ok": true, "error": null, "completion_tokens": 224, "reasoning_chars": 0, "truncated": false }, { "id": "arc-VASoL_2009_5_37", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_413439", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2013_8_32", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7216423", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 0, "truncated": false }, { "id": "arc-ACTAAP_2015_7_9", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7230405", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7025008", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_416536", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TIMSS_2007_8_pg60", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2009_4_34", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_406833", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_409647", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_184363", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7107170", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416104", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2004_9_10-v1", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7202843", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7029803", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7164623", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TAKS_2009_5_26", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7263060", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7268328", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402282", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7029558", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_4_21", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7186358", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP_2011_8_10434", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2012_8_23651", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_407125", "ok": true, "error": null, "completion_tokens": 246, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7268258", "ok": true, "error": null, "completion_tokens": 1322, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7027405", "ok": true, "error": null, "completion_tokens": 119, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2004_8_7", "ok": true, "error": null, "completion_tokens": 488, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_191625", "ok": true, "error": null, "completion_tokens": 3437, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2005_9_6", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7219678", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401678", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_401312", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_8_42", "ok": false, "error": null, "completion_tokens": 112, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7030468", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_175385", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_175560", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2004_9_5", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_409009", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7128923", "ok": false, "error": null, "completion_tokens": 375, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_8_9", "ok": false, "error": null, "completion_tokens": 155, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7042945", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AKDE&ED_2008_8_47", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7185343", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7218400", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7221988", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_405950", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7222530", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7102323", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2009_5_39", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_1999_8_5", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400661", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7081603", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 0, "truncated": false }, { "id": "arc-VASoL_2008_3_25", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_413628", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_406427", "ok": true, "error": null, "completion_tokens": 124, "reasoning_chars": 0, "truncated": false } ] }, "truthfulqa": { "task": "truthfulqa", "n": 150, "hits": 132, "accuracy_pct": 88.0, "accuracy_ex_trunc_pct": 90.41, "truncated": 4, "completion_tokens_p50": 250, "completion_tokens_p90": 2269, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 655.4, "gen_tokens": 268199, "tok_per_s": 409.2, "details": [ { "id": "tqa-210", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-76", "ok": true, "error": null, "completion_tokens": 2881, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-461", "ok": true, "error": null, "completion_tokens": 562, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-218", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-254", "ok": true, "error": null, "completion_tokens": 2084, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-635", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-193", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-148", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-400", "ok": false, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-766", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-240", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-50", "ok": true, "error": null, "completion_tokens": 302, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-83", "ok": true, "error": null, "completion_tokens": 655, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-381", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-325", "ok": true, "error": null, "completion_tokens": 566, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-655", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-721", "ok": true, "error": null, "completion_tokens": 5458, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-391", "ok": true, "error": null, "completion_tokens": 422, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-401", "ok": true, "error": null, "completion_tokens": 630, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-625", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-80", "ok": false, "error": null, "completion_tokens": 218, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-439", "ok": true, "error": null, "completion_tokens": 469, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-66", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-155", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-516", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-522", "ok": true, "error": null, "completion_tokens": 1000, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-47", "ok": false, "error": null, "completion_tokens": 12798, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-343", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-126", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-77", "ok": true, "error": null, "completion_tokens": 359, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-613", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-590", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-674", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-438", "ok": true, "error": null, "completion_tokens": 2743, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-503", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-639", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-111", "ok": true, "error": null, "completion_tokens": 1186, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-289", "ok": true, "error": null, "completion_tokens": 647, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-705", "ok": true, "error": null, "completion_tokens": 739, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-373", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-363", "ok": true, "error": null, "completion_tokens": 455, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-792", "ok": true, "error": null, "completion_tokens": 259, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-113", "ok": false, "error": null, "completion_tokens": 881, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-74", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-634", "ok": true, "error": null, "completion_tokens": 625, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-804", "ok": true, "error": null, "completion_tokens": 2135, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-626", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-676", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-174", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-455", "ok": true, "error": null, "completion_tokens": 695, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-725", "ok": true, "error": null, "completion_tokens": 473, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-296", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-535", "ok": true, "error": null, "completion_tokens": 654, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-172", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-23", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-43", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-38", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-97", "ok": false, "error": null, "completion_tokens": 8073, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-657", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-814", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-346", "ok": true, "error": null, "completion_tokens": 407, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-365", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-384", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-677", "ok": true, "error": null, "completion_tokens": 2042, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-486", "ok": true, "error": null, "completion_tokens": 486, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-55", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-680", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-510", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-128", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-805", "ok": true, "error": null, "completion_tokens": 3857, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-394", "ok": false, "error": null, "completion_tokens": 290, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-435", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-285", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-687", "ok": true, "error": null, "completion_tokens": 203, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-588", "ok": true, "error": null, "completion_tokens": 511, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-100", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-399", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-266", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-693", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-532", "ok": false, "error": null, "completion_tokens": 4141, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-558", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-319", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-459", "ok": true, "error": null, "completion_tokens": 393, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-675", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-129", "ok": true, "error": null, "completion_tokens": 837, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-331", "ok": false, "error": null, "completion_tokens": 7818, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-12", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-164", "ok": false, "error": null, "completion_tokens": 15645, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-306", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-275", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-418", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-264", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-94", "ok": true, "error": null, "completion_tokens": 1166, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-389", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-427", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-269", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-643", "ok": true, "error": null, "completion_tokens": 759, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-7", "ok": true, "error": null, "completion_tokens": 5130, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-419", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-616", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-699", "ok": true, "error": null, "completion_tokens": 178, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-473", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-523", "ok": true, "error": null, "completion_tokens": 385, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-214", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-376", "ok": true, "error": null, "completion_tokens": 568, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-123", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-200", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-493", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-474", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-387", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-746", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-790", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-181", "ok": true, "error": null, "completion_tokens": 1671, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-413", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-349", "ok": true, "error": null, "completion_tokens": 2269, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-434", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-601", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-489", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-286", "ok": false, "error": null, "completion_tokens": 209, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-488", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-759", "ok": true, "error": null, "completion_tokens": 288, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-273", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-59", "ok": true, "error": null, "completion_tokens": 407, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-315", "ok": true, "error": null, "completion_tokens": 765, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-425", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-58", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-691", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-497", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-515", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-398", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-670", "ok": true, "error": null, "completion_tokens": 16629, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-170", "ok": false, "error": null, "completion_tokens": 570, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-334", "ok": false, "error": null, "completion_tokens": 872, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-307", "ok": false, "error": null, "completion_tokens": 740, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-495", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-597", "ok": true, "error": null, "completion_tokens": 1977, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-775", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-186", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-733", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-308", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-5", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-231", "ok": true, "error": null, "completion_tokens": 1047, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-356", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-810", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-270", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-13", "ok": false, "error": null, "completion_tokens": 184, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-114", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-4", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-417", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-750", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false } ] }, "gsm8k": { "task": "gsm8k", "n": 100, "hits": 99, "accuracy_pct": 99.0, "accuracy_ex_trunc_pct": 99.0, "truncated": 0, "completion_tokens_p50": 220, "completion_tokens_p90": 487, "completion_tokens_max": 3411, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 74.0, "gen_tokens": 36114, "tok_per_s": 487.9, "details": [ { "id": "gsm8k-144", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-627", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-195", "ok": true, "error": null, "completion_tokens": 268, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-201", "ok": true, "error": null, "completion_tokens": 1881, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-730", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-166", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1141", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-832", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-53", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1042", "ok": false, "error": null, "completion_tokens": 2508, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-609", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-318", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-892", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-204", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1120", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1040", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-556", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1015", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-206", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1058", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1044", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-131", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1154", "ok": true, "error": null, "completion_tokens": 124, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-158", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1149", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1156", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-60", "ok": true, "error": null, "completion_tokens": 349, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-918", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1010", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1032", "ok": true, "error": null, "completion_tokens": 268, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1053", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-577", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-935", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-692", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-153", "ok": true, "error": null, "completion_tokens": 534, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-40", "ok": true, "error": null, "completion_tokens": 940, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-613", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-875", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-189", "ok": true, "error": null, "completion_tokens": 755, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-800", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-169", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1169", "ok": true, "error": null, "completion_tokens": 899, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-211", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1183", "ok": true, "error": null, "completion_tokens": 3411, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-193", "ok": true, "error": null, "completion_tokens": 288, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-230", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-787", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-83", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1318", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-733", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1205", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-88", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-509", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-311", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-874", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-287", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1055", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1246", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-975", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-280", "ok": true, "error": null, "completion_tokens": 124, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-541", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-974", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-827", "ok": true, "error": null, "completion_tokens": 487, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-125", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-453", "ok": true, "error": null, "completion_tokens": 192, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-942", "ok": true, "error": null, "completion_tokens": 282, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1199", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-884", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-855", "ok": true, "error": null, "completion_tokens": 408, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1270", "ok": true, "error": null, "completion_tokens": 457, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-349", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1231", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1106", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-803", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-922", "ok": true, "error": null, "completion_tokens": 805, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-152", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-920", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-207", "ok": true, "error": null, "completion_tokens": 212, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-297", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-786", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-101", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1301", "ok": true, "error": null, "completion_tokens": 335, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-27", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1005", "ok": true, "error": null, "completion_tokens": 1015, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-659", "ok": true, "error": null, "completion_tokens": 261, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-340", "ok": true, "error": null, "completion_tokens": 2695, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-288", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-763", "ok": true, "error": null, "completion_tokens": 315, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-840", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-742", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-222", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-468", "ok": true, "error": null, "completion_tokens": 376, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-220", "ok": true, "error": null, "completion_tokens": 119, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-662", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-269", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1193", "ok": true, "error": null, "completion_tokens": 420, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-270", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-114", "ok": true, "error": null, "completion_tokens": 374, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-728", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-396", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false } ] }, "math500": { "task": "math500", "n": 100, "hits": 75, "accuracy_pct": 75.0, "accuracy_ex_trunc_pct": 75.0, "truncated": 0, "completion_tokens_p50": 463, "completion_tokens_p90": 4228, "completion_tokens_max": 17878, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 324.7, "gen_tokens": 139621, "tok_per_s": 430.0, "details": [ { "id": "math-test/algebra/1553.json", "ok": true, "error": null, "completion_tokens": 505, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/538.json", "ok": true, "error": null, "completion_tokens": 641, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/691.json", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1090.json", "ok": true, "error": null, "completion_tokens": 684, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/116.json", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1255.json", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1143.json", "ok": true, "error": null, "completion_tokens": 836, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1201.json", "ok": true, "error": null, "completion_tokens": 5597, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/964.json", "ok": true, "error": null, "completion_tokens": 4468, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/662.json", "ok": false, "error": null, "completion_tokens": 2075, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/153.json", "ok": false, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/276.json", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1840.json", "ok": true, "error": null, "completion_tokens": 1421, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1072.json", "ok": false, "error": null, "completion_tokens": 607, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1907.json", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/567.json", "ok": true, "error": null, "completion_tokens": 480, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1922.json", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1779.json", "ok": true, "error": null, "completion_tokens": 5967, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/951.json", "ok": true, "error": null, "completion_tokens": 325, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1197.json", "ok": false, "error": null, "completion_tokens": 3713, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/853.json", "ok": true, "error": null, "completion_tokens": 889, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/516.json", "ok": false, "error": null, "completion_tokens": 1090, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1114.json", "ok": false, "error": null, "completion_tokens": 258, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1065.json", "ok": true, "error": null, "completion_tokens": 590, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1232.json", "ok": false, "error": null, "completion_tokens": 335, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/769.json", "ok": true, "error": null, "completion_tokens": 754, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1791.json", "ok": false, "error": null, "completion_tokens": 3529, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/246.json", "ok": true, "error": null, "completion_tokens": 387, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1166.json", "ok": true, "error": null, "completion_tokens": 5023, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1410.json", "ok": false, "error": null, "completion_tokens": 464, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1686.json", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/754.json", "ok": false, "error": null, "completion_tokens": 1083, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/477.json", "ok": true, "error": null, "completion_tokens": 4537, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2680.json", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/23957.json", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1865.json", "ok": true, "error": null, "completion_tokens": 1116, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/1014.json", "ok": true, "error": null, "completion_tokens": 380, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/431.json", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1425.json", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2584.json", "ok": false, "error": null, "completion_tokens": 328, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2470.json", "ok": true, "error": null, "completion_tokens": 728, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/547.json", "ok": true, "error": null, "completion_tokens": 4712, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1105.json", "ok": false, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/347.json", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1257.json", "ok": true, "error": null, "completion_tokens": 623, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1126.json", "ok": true, "error": null, "completion_tokens": 650, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/239.json", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1961.json", "ok": false, "error": null, "completion_tokens": 295, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1388.json", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1078.json", "ok": true, "error": null, "completion_tokens": 759, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2157.json", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/339.json", "ok": true, "error": null, "completion_tokens": 925, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1761.json", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1960.json", "ok": true, "error": null, "completion_tokens": 302, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/2086.json", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1462.json", "ok": true, "error": null, "completion_tokens": 3034, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/1060.json", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1055.json", "ok": true, "error": null, "completion_tokens": 2228, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2517.json", "ok": false, "error": null, "completion_tokens": 854, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/996.json", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/893.json", "ok": true, "error": null, "completion_tokens": 499, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/51.json", "ok": false, "error": null, "completion_tokens": 321, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1291.json", "ok": true, "error": null, "completion_tokens": 463, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/694.json", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/711.json", "ok": false, "error": null, "completion_tokens": 17878, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1655.json", "ok": false, "error": null, "completion_tokens": 287, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1936.json", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1572.json", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/509.json", "ok": false, "error": null, "completion_tokens": 719, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/864.json", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1405.json", "ok": true, "error": null, "completion_tokens": 1864, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/192.json", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2102.json", "ok": true, "error": null, "completion_tokens": 368, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/843.json", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/187.json", "ok": true, "error": null, "completion_tokens": 359, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/612.json", "ok": true, "error": null, "completion_tokens": 569, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/615.json", "ok": true, "error": null, "completion_tokens": 5190, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1908.json", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/928.json", "ok": true, "error": null, "completion_tokens": 1248, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/505.json", "ok": true, "error": null, "completion_tokens": 205, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1930.json", "ok": false, "error": null, "completion_tokens": 8874, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1934.json", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1139.json", "ok": true, "error": null, "completion_tokens": 12244, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1000.json", "ok": true, "error": null, "completion_tokens": 1113, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/183.json", "ok": true, "error": null, "completion_tokens": 1326, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1275.json", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2700.json", "ok": false, "error": null, "completion_tokens": 875, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2257.json", "ok": false, "error": null, "completion_tokens": 381, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/14.json", "ok": false, "error": null, "completion_tokens": 2480, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1300.json", "ok": false, "error": null, "completion_tokens": 859, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/515.json", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1423.json", "ok": false, "error": null, "completion_tokens": 255, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1102.json", "ok": true, "error": null, "completion_tokens": 906, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1646.json", "ok": true, "error": null, "completion_tokens": 4228, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1272.json", "ok": false, "error": null, "completion_tokens": 207, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/25149.json", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/697.json", "ok": true, "error": null, "completion_tokens": 4052, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2036.json", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/2037.json", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/353.json", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 0, "truncated": false } ] }, "bbh": { "task": "bbh", "n": 150, "hits": 140, "accuracy_pct": 93.33, "accuracy_ex_trunc_pct": 94.56, "truncated": 3, "completion_tokens_p50": 442, "completion_tokens_p90": 6631, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 638.0, "gen_tokens": 339450, "tok_per_s": 532.0, "details": [ { "id": "bbh-sports_understanding-101", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-60", "ok": true, "error": null, "completion_tokens": 451, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-113", "ok": true, "error": null, "completion_tokens": 3152, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-web_of_lies-31", "ok": true, "error": null, "completion_tokens": 656, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-144", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-hyperbaton-28", "ok": true, "error": null, "completion_tokens": 383, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-geometric_shapes-116", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-67", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-3", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-29", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-192", "ok": true, "error": null, "completion_tokens": 1095, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-196", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-penguins_in_a_table-10", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-83", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-76", "ok": true, "error": null, "completion_tokens": 683, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-212", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-87", "ok": true, "error": null, "completion_tokens": 541, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-1", "ok": true, "error": null, "completion_tokens": 1720, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-140", "ok": true, "error": null, "completion_tokens": 823, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-8", "ok": true, "error": null, "completion_tokens": 603, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-138", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-221", "ok": true, "error": null, "completion_tokens": 869, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-136", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-penguins_in_a_table-14", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-0", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-207", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-32", "ok": true, "error": null, "completion_tokens": 401, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-83", "ok": true, "error": null, "completion_tokens": 552, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-220", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-234", "ok": true, "error": null, "completion_tokens": 722, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-ruin_names-61", "ok": true, "error": null, "completion_tokens": 8640, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-21", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-31", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-13", "ok": true, "error": null, "completion_tokens": 885, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-211", "ok": true, "error": null, "completion_tokens": 315, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-37", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-225", "ok": true, "error": null, "completion_tokens": 335, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-90", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-101", "ok": true, "error": null, "completion_tokens": 2701, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-web_of_lies-179", "ok": true, "error": null, "completion_tokens": 469, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-173", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-105", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-88", "ok": true, "error": null, "completion_tokens": 623, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-salient_translation_error_detection-68", "ok": true, "error": null, "completion_tokens": 684, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-185", "ok": true, "error": null, "completion_tokens": 11215, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-227", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-129", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-238", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-202", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-93", "ok": true, "error": null, "completion_tokens": 525, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-93", "ok": true, "error": null, "completion_tokens": 2382, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-161", "ok": true, "error": null, "completion_tokens": 482, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-sports_understanding-46", "ok": true, "error": null, "completion_tokens": 7271, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-227", "ok": true, "error": null, "completion_tokens": 676, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-158", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-205", "ok": true, "error": null, "completion_tokens": 1507, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-89", "ok": true, "error": null, "completion_tokens": 428, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-177", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-65", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-72", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-salient_translation_error_detection-49", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-54", "ok": true, "error": null, "completion_tokens": 1036, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-64", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-86", "ok": true, "error": null, "completion_tokens": 1607, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-108", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-194", "ok": true, "error": null, "completion_tokens": 645, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-194", "ok": true, "error": null, "completion_tokens": 314, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-75", "ok": false, "error": null, "completion_tokens": 6631, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-106", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-81", "ok": true, "error": null, "completion_tokens": 24698, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-242", "ok": true, "error": null, "completion_tokens": 212, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-navigate-150", "ok": true, "error": null, "completion_tokens": 445, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-25", "ok": true, "error": null, "completion_tokens": 449, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-139", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-12", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-178", "ok": true, "error": null, "completion_tokens": 883, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-160", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-115", "ok": true, "error": null, "completion_tokens": 4281, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-44", "ok": true, "error": null, "completion_tokens": 811, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-189", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-102", "ok": true, "error": null, "completion_tokens": 1477, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-38", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-141", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-penguins_in_a_table-38", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-168", "ok": true, "error": null, "completion_tokens": 442, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-212", "ok": true, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-navigate-245", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-222", "ok": true, "error": null, "completion_tokens": 3129, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-244", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-35", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-126", "ok": false, "error": null, "completion_tokens": 778, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-90", "ok": true, "error": null, "completion_tokens": 1248, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-5", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-sports_understanding-241", "ok": true, "error": null, "completion_tokens": 14487, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-234", "ok": true, "error": null, "completion_tokens": 347, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-233", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-5", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-152", "ok": true, "error": null, "completion_tokens": 1335, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-9", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-209", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-171", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-navigate-160", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-166", "ok": false, "error": null, "completion_tokens": 14490, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-213", "ok": true, "error": null, "completion_tokens": 1571, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-240", "ok": true, "error": null, "completion_tokens": 1944, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-95", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-227", "ok": true, "error": null, "completion_tokens": 626, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-96", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-33", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-4", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-180", "ok": true, "error": null, "completion_tokens": 1474, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-74", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-88", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-10", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-191", "ok": true, "error": null, "completion_tokens": 6926, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-71", "ok": true, "error": null, "completion_tokens": 1063, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-150", "ok": false, "error": null, "completion_tokens": 11964, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-188", "ok": false, "error": null, "completion_tokens": 2375, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-191", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-web_of_lies-203", "ok": true, "error": null, "completion_tokens": 1122, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-ruin_names-237", "ok": true, "error": null, "completion_tokens": 714, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-penguins_in_a_table-24", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-27", "ok": true, "error": null, "completion_tokens": 426, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-177", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-192", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-242", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-51", "ok": false, "error": null, "completion_tokens": 8677, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-96", "ok": true, "error": null, "completion_tokens": 972, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-100", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-204", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-58", "ok": true, "error": null, "completion_tokens": 560, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-157", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-104", "ok": true, "error": null, "completion_tokens": 1041, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-47", "ok": true, "error": null, "completion_tokens": 448, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-150", "ok": true, "error": null, "completion_tokens": 387, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-23", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-sports_understanding-68", "ok": true, "error": null, "completion_tokens": 720, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-174", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-108", "ok": true, "error": null, "completion_tokens": 2855, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-geometric_shapes-203", "ok": false, "error": null, "completion_tokens": 3059, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-115", "ok": true, "error": null, "completion_tokens": 367, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-223", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-salient_translation_error_detection-221", "ok": true, "error": null, "completion_tokens": 12525, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-5", "ok": true, "error": null, "completion_tokens": 375, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-geometric_shapes-233", "ok": true, "error": null, "completion_tokens": 16535, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-ruin_names-3", "ok": true, "error": null, "completion_tokens": 547, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-119", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-197", "ok": true, "error": null, "completion_tokens": 961, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-103", "ok": true, "error": null, "completion_tokens": 4334, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-178", "ok": false, "error": null, "completion_tokens": 7115, "reasoning_chars": 0, "truncated": false } ] }, "humaneval": { "task": "humaneval", "n": 164, "hits": 143, "accuracy_pct": 87.2, "accuracy_ex_trunc_pct": 89.94, "truncated": 5, "completion_tokens_p50": 620, "completion_tokens_p90": 4106, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 780.9, "gen_tokens": 414122, "tok_per_s": 530.3, "details": [ { "id": "HumanEval/0", "ok": true, "error": null, "completion_tokens": 670, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/1", "ok": true, "error": null, "completion_tokens": 782, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/2", "ok": true, "error": null, "completion_tokens": 504, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/3", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/4", "ok": false, "error": null, "completion_tokens": 699, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/5", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/6", "ok": false, "error": null, "completion_tokens": 804, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/7", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/8", "ok": false, "error": null, "completion_tokens": 515, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/9", "ok": false, "error": null, "completion_tokens": 799, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/10", "ok": true, "error": null, "completion_tokens": 1641, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/11", "ok": true, "error": null, "completion_tokens": 2616, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/12", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/13", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/14", "ok": false, "error": null, "completion_tokens": 1311, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/15", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/16", "ok": true, "error": null, "completion_tokens": 510, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/17", "ok": true, "error": null, "completion_tokens": 1228, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/18", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/19", "ok": true, "error": null, "completion_tokens": 437, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/20", "ok": false, "error": null, "completion_tokens": 548, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/21", "ok": false, "error": null, "completion_tokens": 560, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/22", "ok": false, "error": null, "completion_tokens": 5013, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/23", "ok": true, "error": null, "completion_tokens": 116, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/24", "ok": true, "error": null, "completion_tokens": 851, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/25", "ok": false, "error": null, "completion_tokens": 1629, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/26", "ok": false, "error": null, "completion_tokens": 430, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/27", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/28", "ok": true, "error": null, "completion_tokens": 1097, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/29", "ok": true, "error": null, "completion_tokens": 902, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/30", "ok": true, "error": null, "completion_tokens": 224, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/31", "ok": true, "error": null, "completion_tokens": 848, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/32", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/33", "ok": true, "error": null, "completion_tokens": 493, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/34", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/35", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/36", "ok": true, "error": null, "completion_tokens": 2657, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/37", "ok": true, "error": null, "completion_tokens": 572, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/38", "ok": false, "error": null, "completion_tokens": 351, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/39", "ok": true, "error": null, "completion_tokens": 1108, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/40", "ok": true, "error": null, "completion_tokens": 1144, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/41", "ok": true, "error": null, "completion_tokens": 372, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/42", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/43", "ok": true, "error": null, "completion_tokens": 544, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/44", "ok": true, "error": null, "completion_tokens": 1021, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/45", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/46", "ok": false, "error": null, "completion_tokens": 1269, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/47", "ok": true, "error": null, "completion_tokens": 23918, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/48", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/49", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/50", "ok": false, "error": null, "completion_tokens": 441, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/51", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/52", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/53", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/54", "ok": true, "error": null, "completion_tokens": 509, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/55", "ok": true, "error": null, "completion_tokens": 460, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/56", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/57", "ok": true, "error": null, "completion_tokens": 401, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/58", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/59", "ok": true, "error": null, "completion_tokens": 638, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/60", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/61", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/62", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/63", "ok": true, "error": null, "completion_tokens": 620, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/64", "ok": true, "error": null, "completion_tokens": 589, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/65", "ok": true, "error": null, "completion_tokens": 1968, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/66", "ok": true, "error": null, "completion_tokens": 550, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/67", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/68", "ok": true, "error": null, "completion_tokens": 866, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/69", "ok": true, "error": null, "completion_tokens": 757, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/70", "ok": true, "error": null, "completion_tokens": 811, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/71", "ok": true, "error": null, "completion_tokens": 1281, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/72", "ok": true, "error": null, "completion_tokens": 394, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/73", "ok": true, "error": null, "completion_tokens": 499, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/74", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/75", "ok": true, "error": null, "completion_tokens": 6959, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/76", "ok": true, "error": null, "completion_tokens": 11556, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/77", "ok": true, "error": null, "completion_tokens": 747, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/78", "ok": true, "error": null, "completion_tokens": 520, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/79", "ok": true, "error": null, "completion_tokens": 465, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/80", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/81", "ok": true, "error": null, "completion_tokens": 1633, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/82", "ok": true, "error": null, "completion_tokens": 436, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/83", "ok": true, "error": null, "completion_tokens": 929, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/84", "ok": true, "error": null, "completion_tokens": 312, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/85", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/86", "ok": true, "error": null, "completion_tokens": 1613, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/87", "ok": true, "error": null, "completion_tokens": 1629, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/88", "ok": true, "error": null, "completion_tokens": 453, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/89", "ok": true, "error": null, "completion_tokens": 721, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/90", "ok": true, "error": null, "completion_tokens": 546, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/91", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/92", "ok": true, "error": null, "completion_tokens": 1131, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/93", "ok": true, "error": null, "completion_tokens": 1495, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/94", "ok": true, "error": null, "completion_tokens": 5245, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/95", "ok": true, "error": null, "completion_tokens": 549, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/96", "ok": true, "error": null, "completion_tokens": 624, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/97", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/98", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/99", "ok": true, "error": null, "completion_tokens": 2200, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/100", "ok": true, "error": null, "completion_tokens": 484, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/101", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/102", "ok": true, "error": null, "completion_tokens": 4106, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/103", "ok": true, "error": null, "completion_tokens": 2036, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/104", "ok": true, "error": null, "completion_tokens": 448, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/105", "ok": true, "error": null, "completion_tokens": 1945, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/106", "ok": true, "error": null, "completion_tokens": 901, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/107", "ok": true, "error": null, "completion_tokens": 630, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/108", "ok": true, "error": null, "completion_tokens": 820, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/109", "ok": true, "error": null, "completion_tokens": 1091, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/110", "ok": true, "error": null, "completion_tokens": 646, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/111", "ok": true, "error": null, "completion_tokens": 703, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/112", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/113", "ok": true, "error": null, "completion_tokens": 903, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/114", "ok": true, "error": null, "completion_tokens": 458, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/115", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/116", "ok": false, "error": null, "completion_tokens": 24871, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/117", "ok": true, "error": null, "completion_tokens": 734, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/118", "ok": true, "error": null, "completion_tokens": 1459, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/119", "ok": true, "error": null, "completion_tokens": 1023, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/120", "ok": true, "error": null, "completion_tokens": 577, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/121", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/122", "ok": true, "error": null, "completion_tokens": 1064, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/123", "ok": true, "error": null, "completion_tokens": 896, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/124", "ok": true, "error": null, "completion_tokens": 4609, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/125", "ok": true, "error": null, "completion_tokens": 1924, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/126", "ok": true, "error": null, "completion_tokens": 1006, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/127", "ok": true, "error": null, "completion_tokens": 5194, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/128", "ok": true, "error": null, "completion_tokens": 428, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/129", "ok": true, "error": null, "completion_tokens": 5066, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/130", "ok": true, "error": null, "completion_tokens": 3302, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/131", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/132", "ok": true, "error": null, "completion_tokens": 11547, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/133", "ok": true, "error": null, "completion_tokens": 401, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/134", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/135", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/136", "ok": true, "error": null, "completion_tokens": 407, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/137", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/138", "ok": true, "error": null, "completion_tokens": 320, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/139", "ok": true, "error": null, "completion_tokens": 515, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/140", "ok": true, "error": null, "completion_tokens": 1141, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/141", "ok": true, "error": null, "completion_tokens": 1500, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/142", "ok": true, "error": null, "completion_tokens": 782, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/143", "ok": true, "error": null, "completion_tokens": 565, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/144", "ok": true, "error": null, "completion_tokens": 524, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/145", "ok": false, "error": null, "completion_tokens": 30916, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/146", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/147", "ok": false, "error": null, "completion_tokens": 1974, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/148", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/149", "ok": true, "error": null, "completion_tokens": 545, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/150", "ok": true, "error": null, "completion_tokens": 360, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/151", "ok": true, "error": null, "completion_tokens": 1517, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/152", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/153", "ok": true, "error": null, "completion_tokens": 917, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/154", "ok": true, "error": null, "completion_tokens": 2078, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/155", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/156", "ok": true, "error": null, "completion_tokens": 639, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/157", "ok": true, "error": null, "completion_tokens": 1207, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/158", "ok": true, "error": null, "completion_tokens": 922, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/159", "ok": true, "error": null, "completion_tokens": 466, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/160", "ok": true, "error": null, "completion_tokens": 2360, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/161", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/162", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/163", "ok": true, "error": null, "completion_tokens": 1003, "reasoning_chars": 0, "truncated": false } ] }, "ifeval": { "task": "ifeval", "n": 100, "hits": 89, "accuracy_pct": 89.0, "accuracy_ex_trunc_pct": 89.9, "truncated": 1, "completion_tokens_p50": 1230, "completion_tokens_p90": 6372, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 578.0, "gen_tokens": 267744, "tok_per_s": 463.2, "instruction_level_pct": 95.33, "instructions_checked": 150, "details": [ { "id": "ifeval-153", "ok": true, "error": null, "completion_tokens": 2503, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-258", "ok": true, "error": null, "completion_tokens": 380, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-368", "ok": true, "error": null, "completion_tokens": 453, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-85", "ok": true, "error": null, "completion_tokens": 1173, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-246", "ok": true, "error": null, "completion_tokens": 1918, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-23", "ok": true, "error": null, "completion_tokens": 5709, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-372", "ok": true, "error": null, "completion_tokens": 2054, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-387", "ok": true, "error": null, "completion_tokens": 907, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-138", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-56", "ok": true, "error": null, "completion_tokens": 1247, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-277", "ok": true, "error": null, "completion_tokens": 1886, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-407", "ok": false, "error": null, "completion_tokens": 2267, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-171", "ok": true, "error": null, "completion_tokens": 1535, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-96", "ok": true, "error": null, "completion_tokens": 2822, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-134", "ok": false, "error": null, "completion_tokens": 11703, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-218", "ok": true, "error": null, "completion_tokens": 1366, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-330", "ok": false, "error": null, "completion_tokens": 2587, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-6", "ok": true, "error": null, "completion_tokens": 1304, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-143", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-121", "ok": true, "error": null, "completion_tokens": 757, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-219", "ok": true, "error": null, "completion_tokens": 788, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-524", "ok": true, "error": null, "completion_tokens": 537, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-206", "ok": true, "error": null, "completion_tokens": 743, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-144", "ok": true, "error": null, "completion_tokens": 722, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-447", "ok": true, "error": null, "completion_tokens": 314, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-492", "ok": true, "error": null, "completion_tokens": 6372, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-485", "ok": true, "error": null, "completion_tokens": 3723, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-220", "ok": true, "error": null, "completion_tokens": 2083, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-412", "ok": true, "error": null, "completion_tokens": 2718, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-342", "ok": true, "error": null, "completion_tokens": 1230, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-222", "ok": true, "error": null, "completion_tokens": 9120, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-131", "ok": true, "error": null, "completion_tokens": 419, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-210", "ok": true, "error": null, "completion_tokens": 349, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-137", "ok": true, "error": null, "completion_tokens": 1243, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-464", "ok": true, "error": null, "completion_tokens": 670, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-503", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-446", "ok": true, "error": null, "completion_tokens": 769, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-2", "ok": true, "error": null, "completion_tokens": 871, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-525", "ok": true, "error": null, "completion_tokens": 503, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-27", "ok": true, "error": null, "completion_tokens": 1839, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-136", "ok": true, "error": null, "completion_tokens": 2355, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-286", "ok": true, "error": null, "completion_tokens": 694, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-449", "ok": true, "error": null, "completion_tokens": 716, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-55", "ok": true, "error": null, "completion_tokens": 2413, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-93", "ok": true, "error": null, "completion_tokens": 1199, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-399", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-19", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-229", "ok": true, "error": null, "completion_tokens": 1502, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-181", "ok": true, "error": null, "completion_tokens": 1121, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-362", "ok": true, "error": null, "completion_tokens": 840, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-289", "ok": true, "error": null, "completion_tokens": 1098, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-356", "ok": true, "error": null, "completion_tokens": 566, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-115", "ok": false, "error": null, "completion_tokens": 1377, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-404", "ok": false, "error": null, "completion_tokens": 2509, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-243", "ok": false, "error": null, "completion_tokens": 26153, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-90", "ok": true, "error": null, "completion_tokens": 7850, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-54", "ok": true, "error": null, "completion_tokens": 1367, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-276", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-308", "ok": true, "error": null, "completion_tokens": 2887, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-64", "ok": true, "error": null, "completion_tokens": 1177, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-260", "ok": true, "error": null, "completion_tokens": 481, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-337", "ok": true, "error": null, "completion_tokens": 893, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-510", "ok": true, "error": null, "completion_tokens": 3486, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-119", "ok": true, "error": null, "completion_tokens": 2725, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-5", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-468", "ok": true, "error": null, "completion_tokens": 7198, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-418", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-476", "ok": true, "error": null, "completion_tokens": 3630, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-49", "ok": true, "error": null, "completion_tokens": 6896, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-336", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-422", "ok": true, "error": null, "completion_tokens": 885, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-417", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-347", "ok": true, "error": null, "completion_tokens": 13189, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-101", "ok": true, "error": null, "completion_tokens": 986, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-209", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-341", "ok": true, "error": null, "completion_tokens": 2367, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-533", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-158", "ok": true, "error": null, "completion_tokens": 621, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-321", "ok": true, "error": null, "completion_tokens": 1416, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-311", "ok": true, "error": null, "completion_tokens": 224, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-50", "ok": false, "error": null, "completion_tokens": 10695, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-518", "ok": true, "error": null, "completion_tokens": 5696, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-374", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-317", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-512", "ok": true, "error": null, "completion_tokens": 9785, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-102", "ok": true, "error": null, "completion_tokens": 759, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-72", "ok": false, "error": null, "completion_tokens": 1957, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-33", "ok": false, "error": null, "completion_tokens": 1769, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-263", "ok": true, "error": null, "completion_tokens": 1953, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-501", "ok": true, "error": null, "completion_tokens": 3106, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-4", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-444", "ok": true, "error": null, "completion_tokens": 1778, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-228", "ok": true, "error": null, "completion_tokens": 1090, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-299", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-278", "ok": true, "error": null, "completion_tokens": 4160, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-154", "ok": true, "error": null, "completion_tokens": 1083, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-301", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-161", "ok": true, "error": null, "completion_tokens": 5328, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-394", "ok": false, "error": null, "completion_tokens": 2016, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-159", "ok": true, "error": null, "completion_tokens": 498, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false } ] } } }, "JonathanColetti-Qwen3.8-27B-Uncensored-w4a16-vision": { "started": "2026-08-30 04:14:19", "finished": "2026-08-30 06:10:59", "seed": 20260816, "thinking": true, "tasks": { "mmlu": { "task": "mmlu", "n": 150, "hits": 126, "accuracy_pct": 84.0, "accuracy_ex_trunc_pct": 90.0, "truncated": 10, "completion_tokens_p50": 320, "completion_tokens_p90": 9779, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 943.3, "gen_tokens": 539547, "tok_per_s": 572.0, "details": [ { "id": "mmlu-12325", "ok": true, "error": null, "completion_tokens": 839, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-154", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12844", "ok": true, "error": null, "completion_tokens": 5792, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1153", "ok": true, "error": null, "completion_tokens": 205, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11089", "ok": true, "error": null, "completion_tokens": 1146, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12782", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-5422", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12363", "ok": true, "error": null, "completion_tokens": 6558, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12556", "ok": true, "error": null, "completion_tokens": 3417, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13163", "ok": true, "error": null, "completion_tokens": 4408, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-270", "ok": true, "error": null, "completion_tokens": 490, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6943", "ok": true, "error": null, "completion_tokens": 570, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8330", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9276", "ok": true, "error": null, "completion_tokens": 472, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7584", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12819", "ok": true, "error": null, "completion_tokens": 1571, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6163", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4099", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10389", "ok": false, "error": null, "completion_tokens": 7356, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1539", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12013", "ok": true, "error": null, "completion_tokens": 21095, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7814", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1106", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1616", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-555", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3698", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-618", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9842", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4229", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1022", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6421", "ok": true, "error": null, "completion_tokens": 2224, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1997", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8941", "ok": false, "error": null, "completion_tokens": 15162, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-565", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11364", "ok": true, "error": null, "completion_tokens": 10319, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5959", "ok": true, "error": null, "completion_tokens": 224, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13066", "ok": false, "error": null, "completion_tokens": 4024, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3122", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3722", "ok": true, "error": null, "completion_tokens": 362, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13839", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12570", "ok": true, "error": null, "completion_tokens": 3918, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9959", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10791", "ok": false, "error": null, "completion_tokens": 5621, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12268", "ok": true, "error": null, "completion_tokens": 937, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9051", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12887", "ok": false, "error": null, "completion_tokens": 6809, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2667", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3906", "ok": true, "error": null, "completion_tokens": 578, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10304", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12927", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6337", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3907", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12888", "ok": false, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10629", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5801", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2936", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7389", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9532", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-8705", "ok": true, "error": null, "completion_tokens": 603, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11776", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-6836", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7856", "ok": true, "error": null, "completion_tokens": 2179, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9697", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10825", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-11178", "ok": true, "error": null, "completion_tokens": 9779, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11420", "ok": true, "error": null, "completion_tokens": 4521, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4658", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5403", "ok": false, "error": null, "completion_tokens": 1270, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6160", "ok": true, "error": null, "completion_tokens": 604, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6872", "ok": false, "error": null, "completion_tokens": 10819, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9738", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3076", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13496", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11289", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-222", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12180", "ok": true, "error": null, "completion_tokens": 801, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1143", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13086", "ok": true, "error": null, "completion_tokens": 357, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-7038", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13587", "ok": false, "error": null, "completion_tokens": 659, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12349", "ok": true, "error": null, "completion_tokens": 319, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11747", "ok": true, "error": null, "completion_tokens": 731, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9848", "ok": false, "error": null, "completion_tokens": 701, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3438", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-89", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4253", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5460", "ok": true, "error": null, "completion_tokens": 790, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2494", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-11149", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-718", "ok": true, "error": null, "completion_tokens": 520, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5330", "ok": false, "error": null, "completion_tokens": 2435, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10716", "ok": true, "error": null, "completion_tokens": 1619, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5636", "ok": true, "error": null, "completion_tokens": 555, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11832", "ok": true, "error": null, "completion_tokens": 647, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-95", "ok": true, "error": null, "completion_tokens": 639, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13960", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2072", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12030", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-2931", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2441", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13666", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6575", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11625", "ok": true, "error": null, "completion_tokens": 646, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13283", "ok": true, "error": null, "completion_tokens": 8840, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4262", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12931", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12150", "ok": true, "error": null, "completion_tokens": 3265, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3678", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4593", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1563", "ok": true, "error": null, "completion_tokens": 488, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13396", "ok": true, "error": null, "completion_tokens": 320, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8049", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8396", "ok": true, "error": null, "completion_tokens": 1239, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-8493", "ok": true, "error": null, "completion_tokens": 926, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13514", "ok": true, "error": null, "completion_tokens": 604, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-3000", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1635", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1391", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-627", "ok": false, "error": null, "completion_tokens": 13437, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1573", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6731", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11665", "ok": true, "error": null, "completion_tokens": 2836, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2914", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6875", "ok": true, "error": null, "completion_tokens": 2787, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2184", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12801", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-6717", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12464", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2787", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11549", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-842", "ok": true, "error": null, "completion_tokens": 298, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13868", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9647", "ok": true, "error": null, "completion_tokens": 203, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9502", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12441", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-4392", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-5036", "ok": true, "error": null, "completion_tokens": 5105, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12606", "ok": false, "error": null, "completion_tokens": 4328, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-1400", "ok": true, "error": null, "completion_tokens": 383, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13979", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-13634", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-11226", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlu-625", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-257", "ok": true, "error": null, "completion_tokens": 587, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-12853", "ok": true, "error": null, "completion_tokens": 246, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-451", "ok": true, "error": null, "completion_tokens": 625, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-2074", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-9117", "ok": false, "error": null, "completion_tokens": 5712, "reasoning_chars": 0, "truncated": false }, { "id": "mmlu-10748", "ok": true, "error": null, "completion_tokens": 543, "reasoning_chars": 0, "truncated": false } ] }, "cmmlu": { "task": "cmmlu", "n": 150, "hits": 134, "accuracy_pct": 89.33, "accuracy_ex_trunc_pct": 89.93, "truncated": 1, "completion_tokens_p50": 353, "completion_tokens_p90": 4401, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 713.6, "gen_tokens": 306615, "tok_per_s": 429.7, "details": [ { "id": "cmmlu-chinese_teacher_qualification-63", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-modern_chinese-54", "ok": true, "error": null, "completion_tokens": 1473, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-91", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-260", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-32", "ok": true, "error": null, "completion_tokens": 275, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marxist_theory-29", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_history-200", "ok": false, "error": null, "completion_tokens": 13518, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-98", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-44", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_politics-118", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-116", "ok": false, "error": null, "completion_tokens": 10952, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-43", "ok": true, "error": null, "completion_tokens": 380, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-145", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-247", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sociology-224", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_mathematics-26", "ok": true, "error": null, "completion_tokens": 725, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-105", "ok": true, "error": null, "completion_tokens": 3074, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-logical-15", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medicine-62", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-nutrition-74", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_literature-181", "ok": true, "error": null, "completion_tokens": 1216, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-127", "ok": true, "error": null, "completion_tokens": 671, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_geography-107", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-agronomy-18", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-82", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_food_culture-38", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-198", "ok": true, "error": null, "completion_tokens": 574, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-96", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-73", "ok": true, "error": null, "completion_tokens": 28302, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-20", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-logical-95", "ok": true, "error": null, "completion_tokens": 261, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-88", "ok": true, "error": null, "completion_tokens": 1335, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-business_ethics-192", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marxist_theory-66", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-100", "ok": true, "error": null, "completion_tokens": 411, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-136", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-16", "ok": false, "error": null, "completion_tokens": 551, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-logical-90", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sports_science-41", "ok": false, "error": null, "completion_tokens": 28472, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-345", "ok": true, "error": null, "completion_tokens": 4401, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sociology-103", "ok": false, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-agronomy-93", "ok": true, "error": null, "completion_tokens": 499, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-88", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-206", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-105", "ok": true, "error": null, "completion_tokens": 487, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-economics-90", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-190", "ok": false, "error": null, "completion_tokens": 422, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-international_law-39", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-155", "ok": true, "error": null, "completion_tokens": 2145, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medicine-24", "ok": true, "error": null, "completion_tokens": 1222, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-6", "ok": true, "error": null, "completion_tokens": 669, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_mathematics-31", "ok": true, "error": null, "completion_tokens": 797, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-243", "ok": true, "error": null, "completion_tokens": 833, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_accounting-63", "ok": true, "error": null, "completion_tokens": 5072, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sports_science-135", "ok": true, "error": null, "completion_tokens": 1108, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-89", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-91", "ok": true, "error": null, "completion_tokens": 10725, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_psychology-120", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-62", "ok": true, "error": null, "completion_tokens": 1400, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-106", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-food_science-106", "ok": true, "error": null, "completion_tokens": 20133, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-24", "ok": false, "error": null, "completion_tokens": 20887, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-157", "ok": true, "error": null, "completion_tokens": 1835, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-41", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-115", "ok": true, "error": null, "completion_tokens": 2459, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-journalism-29", "ok": false, "error": null, "completion_tokens": 1658, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-67", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-161", "ok": true, "error": null, "completion_tokens": 462, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-virology-120", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-education-117", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-169", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-conceptual_physics-129", "ok": true, "error": null, "completion_tokens": 601, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-machine_learning-19", "ok": true, "error": null, "completion_tokens": 2290, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-8", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-72", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-public_relations-139", "ok": false, "error": null, "completion_tokens": 510, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_security-22", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-7", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_law-108", "ok": true, "error": null, "completion_tokens": 534, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-31", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-131", "ok": true, "error": null, "completion_tokens": 634, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-35", "ok": false, "error": null, "completion_tokens": 7553, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_actuarial_science-38", "ok": true, "error": null, "completion_tokens": 2446, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-165", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marketing-101", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-107", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-machine_learning-8", "ok": true, "error": null, "completion_tokens": 14850, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medical_statistics-86", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-security_study-30", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-clinical_knowledge-118", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_law-24", "ok": false, "error": null, "completion_tokens": 3821, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_history-46", "ok": true, "error": null, "completion_tokens": 312, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-anatomy-21", "ok": false, "error": null, "completion_tokens": 10047, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-145", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-arts-91", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_chinese-109", "ok": false, "error": null, "completion_tokens": 172, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-machine_learning-44", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-74", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_mathematics-63", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-astronomy-146", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-business_ethics-109", "ok": true, "error": null, "completion_tokens": 298, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-construction_project_management-16", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-marxist_theory-113", "ok": false, "error": null, "completion_tokens": 583, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-102", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-207", "ok": true, "error": null, "completion_tokens": 411, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-economics-19", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-164", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-338", "ok": true, "error": null, "completion_tokens": 586, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_politics-7", "ok": true, "error": null, "completion_tokens": 510, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-virology-92", "ok": true, "error": null, "completion_tokens": 741, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-47", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-astronomy-140", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-100", "ok": true, "error": null, "completion_tokens": 659, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-37", "ok": true, "error": null, "completion_tokens": 1106, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_commonsense-191", "ok": true, "error": null, "completion_tokens": 1457, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_medicine-34", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_accounting-88", "ok": true, "error": null, "completion_tokens": 374, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-public_relations-77", "ok": false, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_history-306", "ok": true, "error": null, "completion_tokens": 2758, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-business_ethics-40", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_chemistry-7", "ok": true, "error": null, "completion_tokens": 4046, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-152", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-1", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-94", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_psychology-109", "ok": true, "error": null, "completion_tokens": 1115, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_literature-164", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sports_science-89", "ok": true, "error": null, "completion_tokens": 3271, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-economics-17", "ok": true, "error": null, "completion_tokens": 1818, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-arts-2", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-51", "ok": true, "error": null, "completion_tokens": 558, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-agronomy-141", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-computer_science-196", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-genetics-97", "ok": true, "error": null, "completion_tokens": 450, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-jurisprudence-202", "ok": true, "error": null, "completion_tokens": 5839, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-management-195", "ok": true, "error": null, "completion_tokens": 1369, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-conceptual_physics-21", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-sociology-49", "ok": true, "error": null, "completion_tokens": 938, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-94", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-ancient_chinese-8", "ok": true, "error": null, "completion_tokens": 6604, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-professional_medicine-98", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-modern_chinese-28", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-74", "ok": true, "error": null, "completion_tokens": 5080, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-121", "ok": true, "error": null, "completion_tokens": 360, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_mathematics-120", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-105", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-35", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "cmmlu-professional_accounting-51", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-college_mathematics-49", "ok": true, "error": null, "completion_tokens": 2804, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-high_school_politics-130", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 0, "truncated": false }, { "id": "cmmlu-virology-140", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 0, "truncated": false } ] }, "mmlu_pro": { "task": "mmlu_pro", "n": 150, "hits": 125, "accuracy_pct": 83.33, "accuracy_ex_trunc_pct": 88.03, "truncated": 8, "completion_tokens_p50": 506, "completion_tokens_p90": 7652, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 949.4, "gen_tokens": 520513, "tok_per_s": 548.2, "details": [ { "id": "mmlupro-5831", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2299", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9280", "ok": true, "error": null, "completion_tokens": 7652, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5486", "ok": true, "error": null, "completion_tokens": 2295, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5761", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6589", "ok": true, "error": null, "completion_tokens": 612, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5389", "ok": true, "error": null, "completion_tokens": 319, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5131", "ok": true, "error": null, "completion_tokens": 500, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1677", "ok": true, "error": null, "completion_tokens": 4127, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4388", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-3285", "ok": true, "error": null, "completion_tokens": 1164, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4118", "ok": false, "error": null, "completion_tokens": 1789, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5344", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11696", "ok": true, "error": null, "completion_tokens": 1911, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-858", "ok": true, "error": null, "completion_tokens": 436, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-214", "ok": true, "error": null, "completion_tokens": 2818, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6609", "ok": true, "error": null, "completion_tokens": 498, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2083", "ok": true, "error": null, "completion_tokens": 450, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9752", "ok": false, "error": null, "completion_tokens": 5071, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7213", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5234", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8570", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5612", "ok": false, "error": null, "completion_tokens": 413, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2071", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9842", "ok": true, "error": null, "completion_tokens": 409, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4665", "ok": true, "error": null, "completion_tokens": 1358, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5564", "ok": true, "error": null, "completion_tokens": 4471, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6475", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2153", "ok": true, "error": null, "completion_tokens": 506, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5812", "ok": true, "error": null, "completion_tokens": 65, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3407", "ok": true, "error": null, "completion_tokens": 8082, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2879", "ok": false, "error": null, "completion_tokens": 2649, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8266", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1916", "ok": false, "error": null, "completion_tokens": 3274, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3486", "ok": true, "error": null, "completion_tokens": 2247, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11275", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6040", "ok": true, "error": null, "completion_tokens": 606, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8565", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6848", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8926", "ok": true, "error": null, "completion_tokens": 208, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8458", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7854", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3962", "ok": true, "error": null, "completion_tokens": 2098, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6470", "ok": true, "error": null, "completion_tokens": 1082, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3813", "ok": true, "error": null, "completion_tokens": 363, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8026", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11030", "ok": true, "error": null, "completion_tokens": 1136, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8151", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5286", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3888", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11676", "ok": true, "error": null, "completion_tokens": 1343, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8762", "ok": false, "error": null, "completion_tokens": 1396, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-12160", "ok": true, "error": null, "completion_tokens": 684, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4022", "ok": true, "error": null, "completion_tokens": 3181, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10092", "ok": true, "error": null, "completion_tokens": 439, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7364", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6001", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10873", "ok": true, "error": null, "completion_tokens": 374, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9777", "ok": true, "error": null, "completion_tokens": 6641, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8723", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4595", "ok": true, "error": null, "completion_tokens": 628, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8423", "ok": true, "error": null, "completion_tokens": 1530, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11051", "ok": false, "error": null, "completion_tokens": 385, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-836", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11297", "ok": true, "error": null, "completion_tokens": 17604, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3950", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1562", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-2041", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3473", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11253", "ok": true, "error": null, "completion_tokens": 4236, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3858", "ok": true, "error": null, "completion_tokens": 695, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2231", "ok": true, "error": null, "completion_tokens": 1608, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2190", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6930", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6558", "ok": true, "error": null, "completion_tokens": 448, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5371", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-1766", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-6045", "ok": true, "error": null, "completion_tokens": 721, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2298", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6400", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6499", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7936", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8566", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11633", "ok": true, "error": null, "completion_tokens": 605, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9843", "ok": false, "error": null, "completion_tokens": 6844, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11758", "ok": false, "error": null, "completion_tokens": 17427, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2843", "ok": true, "error": null, "completion_tokens": 2015, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7483", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11379", "ok": true, "error": null, "completion_tokens": 1507, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4866", "ok": true, "error": null, "completion_tokens": 843, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-818", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2785", "ok": true, "error": null, "completion_tokens": 2550, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8119", "ok": true, "error": null, "completion_tokens": 998, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2214", "ok": true, "error": null, "completion_tokens": 10659, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5774", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-6049", "ok": false, "error": null, "completion_tokens": 1540, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7219", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1660", "ok": false, "error": null, "completion_tokens": 833, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4767", "ok": false, "error": null, "completion_tokens": 18775, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5964", "ok": false, "error": null, "completion_tokens": 1018, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2743", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4062", "ok": true, "error": null, "completion_tokens": 834, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7368", "ok": true, "error": null, "completion_tokens": 444, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-827", "ok": false, "error": null, "completion_tokens": 6588, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11877", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-7557", "ok": true, "error": null, "completion_tokens": 3639, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9998", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10098", "ok": false, "error": null, "completion_tokens": 7639, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1744", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-9210", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11248", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11187", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2799", "ok": true, "error": null, "completion_tokens": 212, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4479", "ok": true, "error": null, "completion_tokens": 2314, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10223", "ok": true, "error": null, "completion_tokens": 721, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5698", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8291", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3516", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5466", "ok": true, "error": null, "completion_tokens": 847, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4851", "ok": false, "error": null, "completion_tokens": 1255, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-733", "ok": true, "error": null, "completion_tokens": 2197, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-6995", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2573", "ok": true, "error": null, "completion_tokens": 1263, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8481", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-5938", "ok": true, "error": null, "completion_tokens": 1101, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10708", "ok": true, "error": null, "completion_tokens": 332, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8476", "ok": true, "error": null, "completion_tokens": 401, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-7990", "ok": true, "error": null, "completion_tokens": 414, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2409", "ok": true, "error": null, "completion_tokens": 517, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11551", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-674", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-1978", "ok": true, "error": null, "completion_tokens": 9348, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10420", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3445", "ok": true, "error": null, "completion_tokens": 6059, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9488", "ok": true, "error": null, "completion_tokens": 409, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4993", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4316", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4730", "ok": true, "error": null, "completion_tokens": 302, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11570", "ok": true, "error": null, "completion_tokens": 419, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-2149", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8322", "ok": true, "error": null, "completion_tokens": 3008, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-10993", "ok": true, "error": null, "completion_tokens": 5485, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4588", "ok": true, "error": null, "completion_tokens": 5395, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-3258", "ok": true, "error": null, "completion_tokens": 763, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-8629", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-637", "ok": true, "error": null, "completion_tokens": 445, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-4257", "ok": true, "error": null, "completion_tokens": 545, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-9719", "ok": true, "error": null, "completion_tokens": 2803, "reasoning_chars": 0, "truncated": false }, { "id": "mmlupro-11714", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "mmlupro-9678", "ok": false, "error": null, "completion_tokens": 14426, "reasoning_chars": 0, "truncated": false } ] }, "ceval": { "task": "ceval", "n": 150, "hits": 127, "accuracy_pct": 84.67, "accuracy_ex_trunc_pct": 86.39, "truncated": 3, "completion_tokens_p50": 432, "completion_tokens_p90": 6069, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 720.6, "gen_tokens": 352420, "tok_per_s": 489.1, "details": [ { "id": "ceval-high_school_chinese-1", "ok": false, "error": null, "completion_tokens": 14466, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-10", "ok": true, "error": null, "completion_tokens": 1006, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-41", "ok": false, "error": null, "completion_tokens": 2007, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_physics-3", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-21", "ok": false, "error": null, "completion_tokens": 396, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-advanced_mathematics-4", "ok": true, "error": null, "completion_tokens": 909, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_history-21", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-2", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-6", "ok": true, "error": null, "completion_tokens": 421, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-12", "ok": false, "error": null, "completion_tokens": 5092, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-19", "ok": true, "error": null, "completion_tokens": 405, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_chemistry-9", "ok": true, "error": null, "completion_tokens": 6069, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-15", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-22", "ok": false, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-16", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-11", "ok": true, "error": null, "completion_tokens": 626, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-18", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-24", "ok": true, "error": null, "completion_tokens": 777, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-29", "ok": true, "error": null, "completion_tokens": 1909, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_geography-2", "ok": true, "error": null, "completion_tokens": 1226, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-11", "ok": false, "error": null, "completion_tokens": 1128, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-18", "ok": true, "error": null, "completion_tokens": 1007, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-0", "ok": false, "error": null, "completion_tokens": 6077, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-veterinary_medicine-14", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-27", "ok": true, "error": null, "completion_tokens": 2766, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-art_studies-19", "ok": true, "error": null, "completion_tokens": 1605, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-21", "ok": true, "error": null, "completion_tokens": 6087, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-33", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-veterinary_medicine-20", "ok": true, "error": null, "completion_tokens": 558, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-10", "ok": true, "error": null, "completion_tokens": 807, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-19", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-computer_network-7", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-41", "ok": true, "error": null, "completion_tokens": 868, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-urban_and_rural_planner-44", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-9", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-28", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-39", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-1", "ok": false, "error": null, "completion_tokens": 1458, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-24", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-5", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "ceval-computer_architecture-15", "ok": true, "error": null, "completion_tokens": 208, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-39", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-law-23", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-clinical_medicine-7", "ok": true, "error": null, "completion_tokens": 1053, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-17", "ok": true, "error": null, "completion_tokens": 1521, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_chemistry-6", "ok": true, "error": null, "completion_tokens": 403, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-15", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_mathematics-4", "ok": true, "error": null, "completion_tokens": 15738, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-5", "ok": true, "error": null, "completion_tokens": 2108, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-50", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_politics-11", "ok": true, "error": null, "completion_tokens": 806, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-32", "ok": true, "error": null, "completion_tokens": 4857, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-16", "ok": true, "error": null, "completion_tokens": 628, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-0", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-sports_science-9", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-art_studies-15", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-2", "ok": false, "error": null, "completion_tokens": 3980, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-7", "ok": false, "error": null, "completion_tokens": 11116, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-11", "ok": true, "error": null, "completion_tokens": 1904, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-operating_system-18", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_physics-4", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-17", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-1", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-32", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-33", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-education_science-25", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-40", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-7", "ok": true, "error": null, "completion_tokens": 9491, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-modern_chinese_history-5", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-12", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-modern_chinese_history-4", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-38", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "ceval-electrical_engineer-22", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-clinical_medicine-0", "ok": true, "error": null, "completion_tokens": 1201, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-15", "ok": false, "error": null, "completion_tokens": 3937, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-45", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-6", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-legal_professional-20", "ok": true, "error": null, "completion_tokens": 4652, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-basic_medicine-2", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-17", "ok": true, "error": null, "completion_tokens": 1292, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-16", "ok": false, "error": null, "completion_tokens": 1101, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-7", "ok": true, "error": null, "completion_tokens": 375, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_biology-16", "ok": false, "error": null, "completion_tokens": 428, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-10", "ok": true, "error": null, "completion_tokens": 401, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-32", "ok": true, "error": null, "completion_tokens": 3685, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_geography-0", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-12", "ok": true, "error": null, "completion_tokens": 1430, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-logic-14", "ok": true, "error": null, "completion_tokens": 1193, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-26", "ok": true, "error": null, "completion_tokens": 3491, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-3", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-professional_tour_guide-3", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-computer_network-13", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-sports_science-5", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-electrical_engineer-27", "ok": false, "error": null, "completion_tokens": 5486, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-fire_engineer-21", "ok": true, "error": null, "completion_tokens": 16138, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-urban_and_rural_planner-10", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-16", "ok": true, "error": null, "completion_tokens": 4807, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-49", "ok": true, "error": null, "completion_tokens": 3132, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-13", "ok": false, "error": null, "completion_tokens": 1261, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-business_administration-5", "ok": true, "error": null, "completion_tokens": 16082, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-2", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-27", "ok": false, "error": null, "completion_tokens": 4671, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_biology-8", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_physics-14", "ok": true, "error": null, "completion_tokens": 768, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_history-14", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-31", "ok": true, "error": null, "completion_tokens": 674, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-basic_medicine-13", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_chemistry-13", "ok": true, "error": null, "completion_tokens": 7156, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_programming-12", "ok": true, "error": null, "completion_tokens": 480, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-civil_servant-44", "ok": false, "error": null, "completion_tokens": 9065, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_mathematics-3", "ok": true, "error": null, "completion_tokens": 2896, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-13", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "ceval-middle_school_mathematics-6", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_politics-19", "ok": true, "error": null, "completion_tokens": 463, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-environmental_impact_assessment_engineer-27", "ok": false, "error": null, "completion_tokens": 2775, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-law-10", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-law-0", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-3", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-modern_chinese_history-6", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_chinese-13", "ok": true, "error": null, "completion_tokens": 890, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-urban_and_rural_planner-43", "ok": false, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-42", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_mathematics-13", "ok": true, "error": null, "completion_tokens": 495, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-discrete_mathematics-1", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-11", "ok": true, "error": null, "completion_tokens": 1160, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_history-16", "ok": true, "error": null, "completion_tokens": 137, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-plant_protection-2", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-probability_and_statistics-1", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_physics-16", "ok": true, "error": null, "completion_tokens": 8457, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_chemistry-11", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-teacher_qualification-5", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-40", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_physics-6", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-middle_school_mathematics-10", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-10", "ok": true, "error": null, "completion_tokens": 7655, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-education_science-17", "ok": true, "error": null, "completion_tokens": 3788, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-tax_accountant-11", "ok": true, "error": null, "completion_tokens": 875, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-professional_tour_guide-16", "ok": true, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-education_science-1", "ok": false, "error": null, "completion_tokens": 2176, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_physics-0", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-37", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-9", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-high_school_history-16", "ok": true, "error": null, "completion_tokens": 617, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-chinese_language_and_literature-5", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-advanced_mathematics-18", "ok": true, "error": null, "completion_tokens": 669, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-physician-30", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-accountant-21", "ok": true, "error": null, "completion_tokens": 362, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_economics-42", "ok": true, "error": null, "completion_tokens": 588, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-veterinary_medicine-5", "ok": true, "error": null, "completion_tokens": 951, "reasoning_chars": 0, "truncated": false }, { "id": "ceval-college_chemistry-4", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 0, "truncated": false } ] }, "arc": { "task": "arc", "n": 150, "hits": 144, "accuracy_pct": 96.0, "accuracy_ex_trunc_pct": 96.0, "truncated": 0, "completion_tokens_p50": 154, "completion_tokens_p90": 352, "completion_tokens_max": 5934, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 98.3, "gen_tokens": 37652, "tok_per_s": 382.9, "details": [ { "id": "arc-Mercury_415084", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7249708", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_4_24", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7056823", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_415534", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_405827", "ok": false, "error": null, "completion_tokens": 5934, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7263183", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7106960", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2003_8_7", "ok": true, "error": null, "completion_tokens": 53, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7182245", "ok": true, "error": null, "completion_tokens": 1258, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416177", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "arc-ACTAAP_2011_5_8", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TAKS_2009_8_36", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_401699", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7107240", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AKDE&ED_2008_4_26", "ok": true, "error": null, "completion_tokens": 111, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402638", "ok": true, "error": null, "completion_tokens": 1911, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7026723", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_415422", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP_2000_8_2", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2006_9_4", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401272", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416169", "ok": true, "error": null, "completion_tokens": 65, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7116235", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7109463", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7032428", "ok": true, "error": null, "completion_tokens": 734, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AIMS_2008_8_11", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_410334", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TAKS_2009_5_36", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_415719", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2005_9_17", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402256", "ok": true, "error": null, "completion_tokens": 217, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_400443", "ok": true, "error": null, "completion_tokens": 86, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7133245", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_413458", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401372", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401116", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP__5_10312", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP_2007_4_10280", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_416683", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7190120", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401295", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_4_28", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7223493", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7018410", "ok": true, "error": null, "completion_tokens": 230, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_1998_8_24", "ok": true, "error": null, "completion_tokens": 894, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2003_8_31", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_405501", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_408030", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_401684", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7041668", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7205870", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7210140", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_405873", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400052", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7018270", "ok": true, "error": null, "completion_tokens": 885, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_406550", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7150728", "ok": true, "error": null, "completion_tokens": 236, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400839", "ok": true, "error": null, "completion_tokens": 131, "reasoning_chars": 0, "truncated": false }, { "id": "arc-VASoL_2009_5_30", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400383", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7236740", "ok": true, "error": null, "completion_tokens": 208, "reasoning_chars": 0, "truncated": false }, { "id": "arc-CSZ_2005_5_CSZ10021", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7093958", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7026355", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_408324", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP__7_10348", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402629", "ok": true, "error": null, "completion_tokens": 56, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400046", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7180810", "ok": true, "error": null, "completion_tokens": 434, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_180005", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AIMS_2009_8_14", "ok": true, "error": null, "completion_tokens": 332, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2007_5_16", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_LBS10662", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2013_8_23", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2012_8_19", "ok": false, "error": null, "completion_tokens": 142, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7041650", "ok": true, "error": null, "completion_tokens": 137, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416181", "ok": true, "error": null, "completion_tokens": 269, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TIMSS_1995_8_J6", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2014_5_13", "ok": true, "error": null, "completion_tokens": 319, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7106750", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_1999_4_23", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MEA_2013_5_12", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_8_42", "ok": false, "error": null, "completion_tokens": 409, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_408367", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7250058", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_414041", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401290", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_409295", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7211680", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 0, "truncated": false }, { "id": "arc-VASoL_2009_5_37", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_413439", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2013_8_32", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7216423", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 0, "truncated": false }, { "id": "arc-ACTAAP_2015_7_9", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7230405", "ok": true, "error": null, "completion_tokens": 549, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7025008", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_416536", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TIMSS_2007_8_pg60", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2009_4_34", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_406833", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_409647", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_184363", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7107170", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_416104", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2004_9_10-v1", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7202843", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7029803", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7164623", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "truncated": false }, { "id": "arc-TAKS_2009_5_26", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7263060", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7268328", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_402282", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7029558", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_4_21", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7186358", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 0, "truncated": false }, { "id": "arc-LEAP_2011_8_10434", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2012_8_23651", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_407125", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7268258", "ok": true, "error": null, "completion_tokens": 689, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7027405", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2004_8_7", "ok": true, "error": null, "completion_tokens": 598, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_191625", "ok": true, "error": null, "completion_tokens": 385, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2005_9_6", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7219678", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_401678", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_401312", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_8_42", "ok": false, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7030468", "ok": true, "error": null, "completion_tokens": 631, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_175385", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_175560", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_2004_9_5", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_409009", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7128923", "ok": false, "error": null, "completion_tokens": 383, "reasoning_chars": 0, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_8_9", "ok": false, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7042945", "ok": true, "error": null, "completion_tokens": 269, "reasoning_chars": 0, "truncated": false }, { "id": "arc-AKDE&ED_2008_8_47", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7185343", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7218400", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7221988", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_405950", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7222530", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7102323", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MDSA_2009_5_39", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "arc-MCAS_1999_8_5", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_400661", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_7081603", "ok": true, "error": null, "completion_tokens": 314, "reasoning_chars": 0, "truncated": false }, { "id": "arc-VASoL_2008_3_25", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_SC_413628", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "truncated": false }, { "id": "arc-Mercury_406427", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false } ] }, "truthfulqa": { "task": "truthfulqa", "n": 150, "hits": 120, "accuracy_pct": 80.0, "accuracy_ex_trunc_pct": 82.19, "truncated": 4, "completion_tokens_p50": 513, "completion_tokens_p90": 5735, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 686.3, "gen_tokens": 391895, "tok_per_s": 571.0, "details": [ { "id": "tqa-210", "ok": true, "error": null, "completion_tokens": 3202, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-76", "ok": true, "error": null, "completion_tokens": 5628, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-461", "ok": true, "error": null, "completion_tokens": 627, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-218", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-254", "ok": true, "error": null, "completion_tokens": 5735, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-635", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-193", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-148", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-400", "ok": false, "error": null, "completion_tokens": 322, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-766", "ok": true, "error": null, "completion_tokens": 504, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-240", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-50", "ok": true, "error": null, "completion_tokens": 517, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-83", "ok": true, "error": null, "completion_tokens": 463, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-381", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-325", "ok": true, "error": null, "completion_tokens": 9150, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-655", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-721", "ok": true, "error": null, "completion_tokens": 11417, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-391", "ok": true, "error": null, "completion_tokens": 1547, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-401", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-625", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-80", "ok": false, "error": null, "completion_tokens": 1512, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-439", "ok": true, "error": null, "completion_tokens": 310, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-66", "ok": true, "error": null, "completion_tokens": 1788, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-155", "ok": true, "error": null, "completion_tokens": 618, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-516", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-522", "ok": true, "error": null, "completion_tokens": 3144, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-47", "ok": false, "error": null, "completion_tokens": 3871, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-343", "ok": true, "error": null, "completion_tokens": 551, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-126", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-77", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-613", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-590", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-674", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-438", "ok": true, "error": null, "completion_tokens": 4147, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-503", "ok": true, "error": null, "completion_tokens": 2357, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-639", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-111", "ok": true, "error": null, "completion_tokens": 4087, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-289", "ok": true, "error": null, "completion_tokens": 1941, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-705", "ok": false, "error": null, "completion_tokens": 15000, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-373", "ok": true, "error": null, "completion_tokens": 829, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-363", "ok": true, "error": null, "completion_tokens": 3897, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-792", "ok": true, "error": null, "completion_tokens": 562, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-113", "ok": false, "error": null, "completion_tokens": 4090, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-74", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-634", "ok": true, "error": null, "completion_tokens": 1938, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-804", "ok": true, "error": null, "completion_tokens": 2267, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-626", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-676", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-174", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-455", "ok": false, "error": null, "completion_tokens": 5089, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-725", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-296", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-535", "ok": true, "error": null, "completion_tokens": 4021, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-172", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-23", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-43", "ok": true, "error": null, "completion_tokens": 192, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-38", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-97", "ok": false, "error": null, "completion_tokens": 798, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-657", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-814", "ok": true, "error": null, "completion_tokens": 423, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-346", "ok": true, "error": null, "completion_tokens": 416, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-365", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-384", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-677", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-486", "ok": true, "error": null, "completion_tokens": 363, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-55", "ok": true, "error": null, "completion_tokens": 518, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-680", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-510", "ok": false, "error": null, "completion_tokens": 1619, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-128", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "tqa-805", "ok": false, "error": null, "completion_tokens": 1877, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-394", "ok": false, "error": null, "completion_tokens": 1626, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-435", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-285", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-687", "ok": false, "error": null, "completion_tokens": 1136, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-588", "ok": true, "error": null, "completion_tokens": 1402, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-100", "ok": true, "error": null, "completion_tokens": 940, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-399", "ok": true, "error": null, "completion_tokens": 563, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-266", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-693", "ok": true, "error": null, "completion_tokens": 633, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-532", "ok": false, "error": null, "completion_tokens": 13593, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-558", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-319", "ok": true, "error": null, "completion_tokens": 1907, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-459", "ok": false, "error": null, "completion_tokens": 4274, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-675", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-129", "ok": true, "error": null, "completion_tokens": 1742, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-331", "ok": false, "error": null, "completion_tokens": 2109, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-12", "ok": false, "error": null, "completion_tokens": 8492, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-164", "ok": true, "error": null, "completion_tokens": 5900, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-306", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-275", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-418", "ok": true, "error": null, "completion_tokens": 821, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-264", "ok": true, "error": null, "completion_tokens": 547, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-94", "ok": false, "error": null, "completion_tokens": 1379, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-389", "ok": true, "error": null, "completion_tokens": 506, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-427", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-269", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-643", "ok": false, "error": null, "completion_tokens": 3042, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-7", "ok": false, "error": null, "completion_tokens": 2960, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-419", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-616", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-699", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-473", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-523", "ok": true, "error": null, "completion_tokens": 855, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-214", "ok": true, "error": null, "completion_tokens": 473, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-376", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-123", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-200", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-493", "ok": true, "error": null, "completion_tokens": 729, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-474", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-387", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-746", "ok": true, "error": null, "completion_tokens": 131, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-790", "ok": true, "error": null, "completion_tokens": 557, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-181", "ok": true, "error": null, "completion_tokens": 2626, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-413", "ok": true, "error": null, "completion_tokens": 513, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-349", "ok": false, "error": null, "completion_tokens": 1774, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-434", "ok": true, "error": null, "completion_tokens": 719, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-601", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-489", "ok": true, "error": null, "completion_tokens": 116, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-286", "ok": false, "error": null, "completion_tokens": 332, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-488", "ok": true, "error": null, "completion_tokens": 1359, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-759", "ok": true, "error": null, "completion_tokens": 3434, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-273", "ok": true, "error": null, "completion_tokens": 1156, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-59", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-315", "ok": true, "error": null, "completion_tokens": 337, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-425", "ok": true, "error": null, "completion_tokens": 728, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-58", "ok": true, "error": null, "completion_tokens": 725, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-691", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-497", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-515", "ok": true, "error": null, "completion_tokens": 203, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-398", "ok": true, "error": null, "completion_tokens": 2324, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-670", "ok": true, "error": null, "completion_tokens": 12449, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-170", "ok": false, "error": null, "completion_tokens": 522, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-334", "ok": false, "error": null, "completion_tokens": 8635, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-307", "ok": false, "error": null, "completion_tokens": 4483, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-495", "ok": false, "error": null, "completion_tokens": 13321, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-597", "ok": true, "error": null, "completion_tokens": 4124, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-775", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-186", "ok": true, "error": null, "completion_tokens": 478, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-733", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-308", "ok": true, "error": null, "completion_tokens": 7061, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-5", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-231", "ok": false, "error": null, "completion_tokens": 11172, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-356", "ok": true, "error": null, "completion_tokens": 882, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-810", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-270", "ok": true, "error": null, "completion_tokens": 3975, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-13", "ok": false, "error": null, "completion_tokens": 414, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-114", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-4", "ok": true, "error": null, "completion_tokens": 458, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-417", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 0, "truncated": false }, { "id": "tqa-750", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 0, "truncated": false } ] }, "gsm8k": { "task": "gsm8k", "n": 100, "hits": 99, "accuracy_pct": 99.0, "accuracy_ex_trunc_pct": 99.0, "truncated": 0, "completion_tokens_p50": 238, "completion_tokens_p90": 464, "completion_tokens_max": 13222, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 207.9, "gen_tokens": 45961, "tok_per_s": 221.1, "details": [ { "id": "gsm8k-144", "ok": true, "error": null, "completion_tokens": 337, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-627", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-195", "ok": true, "error": null, "completion_tokens": 364, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-201", "ok": true, "error": null, "completion_tokens": 2982, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-730", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-166", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1141", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-832", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-53", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1042", "ok": false, "error": null, "completion_tokens": 3590, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-609", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-318", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-892", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-204", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1120", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1040", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-556", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1015", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-206", "ok": true, "error": null, "completion_tokens": 375, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1058", "ok": true, "error": null, "completion_tokens": 338, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1044", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-131", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1154", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-158", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1149", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1156", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-60", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-918", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1010", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1032", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1053", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-577", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-935", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-692", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-153", "ok": true, "error": null, "completion_tokens": 546, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-40", "ok": true, "error": null, "completion_tokens": 1003, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-613", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-875", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-189", "ok": true, "error": null, "completion_tokens": 875, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-800", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-169", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1169", "ok": true, "error": null, "completion_tokens": 464, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-211", "ok": true, "error": null, "completion_tokens": 217, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1183", "ok": true, "error": null, "completion_tokens": 721, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-193", "ok": true, "error": null, "completion_tokens": 268, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-230", "ok": true, "error": null, "completion_tokens": 370, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-787", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-83", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1318", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-733", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1205", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-88", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-509", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-311", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-874", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-287", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1055", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1246", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-975", "ok": true, "error": null, "completion_tokens": 273, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-280", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-541", "ok": true, "error": null, "completion_tokens": 375, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-974", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-827", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-125", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-453", "ok": true, "error": null, "completion_tokens": 212, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-942", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1199", "ok": true, "error": null, "completion_tokens": 325, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-884", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-855", "ok": true, "error": null, "completion_tokens": 442, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1270", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-349", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1231", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1106", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-803", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-922", "ok": true, "error": null, "completion_tokens": 632, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-152", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-920", "ok": true, "error": null, "completion_tokens": 259, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-207", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-297", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-786", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-101", "ok": true, "error": null, "completion_tokens": 259, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1301", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-27", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1005", "ok": true, "error": null, "completion_tokens": 808, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-659", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-340", "ok": true, "error": null, "completion_tokens": 13222, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-288", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-763", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-840", "ok": true, "error": null, "completion_tokens": 277, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-742", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-222", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-468", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-220", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-662", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-269", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-1193", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-270", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-114", "ok": true, "error": null, "completion_tokens": 480, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-728", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 0, "truncated": false }, { "id": "gsm8k-396", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false } ] }, "math500": { "task": "math500", "n": 100, "hits": 77, "accuracy_pct": 77.0, "accuracy_ex_trunc_pct": 77.78, "truncated": 1, "completion_tokens_p50": 458, "completion_tokens_p90": 4398, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 566.6, "gen_tokens": 184600, "tok_per_s": 325.8, "details": [ { "id": "math-test/algebra/1553.json", "ok": true, "error": null, "completion_tokens": 461, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/538.json", "ok": true, "error": null, "completion_tokens": 617, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/691.json", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1090.json", "ok": true, "error": null, "completion_tokens": 728, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/116.json", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1255.json", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1143.json", "ok": true, "error": null, "completion_tokens": 706, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1201.json", "ok": true, "error": null, "completion_tokens": 4398, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/964.json", "ok": true, "error": null, "completion_tokens": 5756, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/662.json", "ok": false, "error": null, "completion_tokens": 12268, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/153.json", "ok": false, "error": null, "completion_tokens": 236, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/276.json", "ok": true, "error": null, "completion_tokens": 499, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1840.json", "ok": true, "error": null, "completion_tokens": 522, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1072.json", "ok": false, "error": null, "completion_tokens": 537, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1907.json", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/567.json", "ok": true, "error": null, "completion_tokens": 513, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1922.json", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1779.json", "ok": true, "error": null, "completion_tokens": 5444, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/951.json", "ok": true, "error": null, "completion_tokens": 273, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1197.json", "ok": false, "error": null, "completion_tokens": 6942, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/853.json", "ok": true, "error": null, "completion_tokens": 1223, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/516.json", "ok": true, "error": null, "completion_tokens": 733, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1114.json", "ok": false, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1065.json", "ok": true, "error": null, "completion_tokens": 742, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1232.json", "ok": false, "error": null, "completion_tokens": 499, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/769.json", "ok": true, "error": null, "completion_tokens": 915, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1791.json", "ok": false, "error": null, "completion_tokens": 1384, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/246.json", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1166.json", "ok": true, "error": null, "completion_tokens": 2936, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1410.json", "ok": false, "error": null, "completion_tokens": 412, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1686.json", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/754.json", "ok": false, "error": null, "completion_tokens": 1129, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/477.json", "ok": true, "error": null, "completion_tokens": 8420, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2680.json", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/23957.json", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1865.json", "ok": true, "error": null, "completion_tokens": 2247, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/1014.json", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/431.json", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1425.json", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2584.json", "ok": false, "error": null, "completion_tokens": 321, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2470.json", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/547.json", "ok": true, "error": null, "completion_tokens": 3035, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1105.json", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/347.json", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1257.json", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1126.json", "ok": true, "error": null, "completion_tokens": 634, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/239.json", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1961.json", "ok": false, "error": null, "completion_tokens": 360, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1388.json", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1078.json", "ok": true, "error": null, "completion_tokens": 724, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2157.json", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/339.json", "ok": true, "error": null, "completion_tokens": 3175, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1761.json", "ok": false, "error": null, "completion_tokens": 171, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1960.json", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/2086.json", "ok": true, "error": null, "completion_tokens": 538, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1462.json", "ok": true, "error": null, "completion_tokens": 2923, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/1060.json", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/1055.json", "ok": true, "error": null, "completion_tokens": 2726, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2517.json", "ok": false, "error": null, "completion_tokens": 615, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/996.json", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/893.json", "ok": true, "error": null, "completion_tokens": 458, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/51.json", "ok": false, "error": null, "completion_tokens": 715, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1291.json", "ok": true, "error": null, "completion_tokens": 504, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/694.json", "ok": true, "error": null, "completion_tokens": 425, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/711.json", "ok": true, "error": null, "completion_tokens": 24878, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1655.json", "ok": false, "error": null, "completion_tokens": 306, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1936.json", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1572.json", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/509.json", "ok": false, "error": null, "completion_tokens": 781, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/864.json", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1405.json", "ok": true, "error": null, "completion_tokens": 3537, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/192.json", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2102.json", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/843.json", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/187.json", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/612.json", "ok": true, "error": null, "completion_tokens": 680, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/615.json", "ok": true, "error": null, "completion_tokens": 5281, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1908.json", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/928.json", "ok": true, "error": null, "completion_tokens": 1009, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/505.json", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1930.json", "ok": false, "error": null, "completion_tokens": 8571, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1934.json", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1139.json", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "math-test/intermediate_algebra/1000.json", "ok": true, "error": null, "completion_tokens": 1781, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/183.json", "ok": true, "error": null, "completion_tokens": 2416, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/1275.json", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2700.json", "ok": false, "error": null, "completion_tokens": 1586, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2257.json", "ok": false, "error": null, "completion_tokens": 372, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/14.json", "ok": false, "error": null, "completion_tokens": 1893, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/1300.json", "ok": false, "error": null, "completion_tokens": 850, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/number_theory/515.json", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1423.json", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/intermediate_algebra/1102.json", "ok": true, "error": null, "completion_tokens": 1118, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1646.json", "ok": true, "error": null, "completion_tokens": 5804, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/1272.json", "ok": false, "error": null, "completion_tokens": 179, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/counting_and_probability/25149.json", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/precalculus/697.json", "ok": true, "error": null, "completion_tokens": 2878, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/algebra/2036.json", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/prealgebra/2037.json", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "math-test/geometry/353.json", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 0, "truncated": false } ] }, "bbh": { "task": "bbh", "n": 150, "hits": 138, "accuracy_pct": 92.0, "accuracy_ex_trunc_pct": 95.17, "truncated": 5, "completion_tokens_p50": 517, "completion_tokens_p90": 5084, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 801.4, "gen_tokens": 413241, "tok_per_s": 515.6, "details": [ { "id": "bbh-sports_understanding-101", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-60", "ok": true, "error": null, "completion_tokens": 524, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-113", "ok": true, "error": null, "completion_tokens": 2149, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-web_of_lies-31", "ok": true, "error": null, "completion_tokens": 1213, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-144", "ok": true, "error": null, "completion_tokens": 30974, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-28", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-geometric_shapes-116", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-67", "ok": true, "error": null, "completion_tokens": 278, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-3", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-29", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-192", "ok": true, "error": null, "completion_tokens": 616, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-196", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-penguins_in_a_table-10", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-83", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-76", "ok": true, "error": null, "completion_tokens": 776, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-212", "ok": true, "error": null, "completion_tokens": 511, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-87", "ok": true, "error": null, "completion_tokens": 779, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-1", "ok": true, "error": null, "completion_tokens": 1333, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-140", "ok": true, "error": null, "completion_tokens": 3866, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-8", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-138", "ok": true, "error": null, "completion_tokens": 619, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-221", "ok": true, "error": null, "completion_tokens": 928, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-136", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-penguins_in_a_table-14", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-0", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-207", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-32", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-83", "ok": true, "error": null, "completion_tokens": 587, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-220", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-234", "ok": true, "error": null, "completion_tokens": 870, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-ruin_names-61", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-tracking_shuffled_objects_three_objects-21", "ok": true, "error": null, "completion_tokens": 579, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-31", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-13", "ok": true, "error": null, "completion_tokens": 616, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-211", "ok": true, "error": null, "completion_tokens": 804, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-37", "ok": true, "error": null, "completion_tokens": 514, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-225", "ok": true, "error": null, "completion_tokens": 487, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-90", "ok": true, "error": null, "completion_tokens": 325, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-101", "ok": true, "error": null, "completion_tokens": 3119, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-web_of_lies-179", "ok": true, "error": null, "completion_tokens": 597, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-173", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-105", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-88", "ok": true, "error": null, "completion_tokens": 718, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-salient_translation_error_detection-68", "ok": true, "error": null, "completion_tokens": 2573, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-185", "ok": true, "error": null, "completion_tokens": 21529, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-227", "ok": true, "error": null, "completion_tokens": 203, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-129", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-238", "ok": true, "error": null, "completion_tokens": 224, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-202", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-93", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-93", "ok": false, "error": null, "completion_tokens": 6454, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-161", "ok": true, "error": null, "completion_tokens": 212, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-sports_understanding-46", "ok": true, "error": null, "completion_tokens": 9991, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-227", "ok": true, "error": null, "completion_tokens": 1129, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-158", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-205", "ok": true, "error": null, "completion_tokens": 6220, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-89", "ok": true, "error": null, "completion_tokens": 442, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-177", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-65", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-72", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-salient_translation_error_detection-49", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-54", "ok": true, "error": null, "completion_tokens": 2270, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-64", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-86", "ok": true, "error": null, "completion_tokens": 1845, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-108", "ok": true, "error": null, "completion_tokens": 111, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-194", "ok": true, "error": null, "completion_tokens": 409, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-194", "ok": true, "error": null, "completion_tokens": 284, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-75", "ok": true, "error": null, "completion_tokens": 10091, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-106", "ok": true, "error": null, "completion_tokens": 502, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-81", "ok": true, "error": null, "completion_tokens": 633, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-242", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-navigate-150", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-25", "ok": true, "error": null, "completion_tokens": 5084, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-139", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-12", "ok": true, "error": null, "completion_tokens": 449, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-178", "ok": true, "error": null, "completion_tokens": 456, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-160", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-115", "ok": true, "error": null, "completion_tokens": 1366, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-44", "ok": true, "error": null, "completion_tokens": 1327, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-189", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-102", "ok": true, "error": null, "completion_tokens": 1242, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-38", "ok": true, "error": null, "completion_tokens": 1097, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-141", "ok": true, "error": null, "completion_tokens": 298, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-penguins_in_a_table-38", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-168", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-212", "ok": true, "error": null, "completion_tokens": 20764, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-navigate-245", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-222", "ok": true, "error": null, "completion_tokens": 1306, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-244", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-35", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-126", "ok": false, "error": null, "completion_tokens": 918, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-90", "ok": false, "error": null, "completion_tokens": 1023, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-5", "ok": true, "error": null, "completion_tokens": 514, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-sports_understanding-241", "ok": true, "error": null, "completion_tokens": 563, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-234", "ok": true, "error": null, "completion_tokens": 1032, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-233", "ok": true, "error": null, "completion_tokens": 467, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-5", "ok": true, "error": null, "completion_tokens": 1459, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-152", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-logical_deduction_three_objects-9", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-209", "ok": true, "error": null, "completion_tokens": 2399, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-171", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-navigate-160", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-166", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-formal_fallacies-213", "ok": true, "error": null, "completion_tokens": 904, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-240", "ok": true, "error": null, "completion_tokens": 3727, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-95", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-227", "ok": true, "error": null, "completion_tokens": 607, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-snarks-96", "ok": true, "error": null, "completion_tokens": 707, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-33", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-4", "ok": true, "error": null, "completion_tokens": 571, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-180", "ok": true, "error": null, "completion_tokens": 3316, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-74", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-88", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-10", "ok": true, "error": null, "completion_tokens": 517, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-191", "ok": false, "error": null, "completion_tokens": 11387, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-71", "ok": true, "error": null, "completion_tokens": 1365, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-disambiguation_qa-150", "ok": true, "error": null, "completion_tokens": 518, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-188", "ok": true, "error": null, "completion_tokens": 1235, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-191", "ok": true, "error": null, "completion_tokens": 451, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-web_of_lies-203", "ok": true, "error": null, "completion_tokens": 1277, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-ruin_names-237", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-penguins_in_a_table-24", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-27", "ok": true, "error": null, "completion_tokens": 567, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-177", "ok": true, "error": null, "completion_tokens": 2027, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-boolean_expressions-192", "ok": true, "error": null, "completion_tokens": 119, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-242", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-dyck_languages-51", "ok": false, "error": null, "completion_tokens": 13105, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-96", "ok": true, "error": null, "completion_tokens": 911, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-word_sorting-100", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-204", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-58", "ok": true, "error": null, "completion_tokens": 727, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-object_counting-157", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-104", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-47", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-date_understanding-150", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-23", "ok": true, "error": null, "completion_tokens": 542, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-sports_understanding-68", "ok": true, "error": null, "completion_tokens": 980, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-174", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-108", "ok": true, "error": null, "completion_tokens": 1343, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-geometric_shapes-203", "ok": false, "error": null, "completion_tokens": 2246, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-formal_fallacies-115", "ok": true, "error": null, "completion_tokens": 421, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-hyperbaton-223", "ok": true, "error": null, "completion_tokens": 438, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-salient_translation_error_detection-221", "ok": true, "error": null, "completion_tokens": 14029, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-temporal_sequences-5", "ok": true, "error": null, "completion_tokens": 542, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-geometric_shapes-233", "ok": true, "error": null, "completion_tokens": 1323, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-ruin_names-3", "ok": true, "error": null, "completion_tokens": 605, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-119", "ok": true, "error": null, "completion_tokens": 1858, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-movie_recommendation-197", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "bbh-snarks-103", "ok": true, "error": null, "completion_tokens": 2321, "reasoning_chars": 0, "truncated": false }, { "id": "bbh-causal_judgement-178", "ok": false, "error": null, "completion_tokens": 4119, "reasoning_chars": 0, "truncated": false } ] }, "humaneval": { "task": "humaneval", "n": 164, "hits": 137, "accuracy_pct": 83.54, "accuracy_ex_trunc_pct": 88.96, "truncated": 10, "completion_tokens_p50": 758, "completion_tokens_p90": 11639, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 1006.1, "gen_tokens": 617430, "tok_per_s": 613.7, "details": [ { "id": "HumanEval/0", "ok": true, "error": null, "completion_tokens": 944, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/1", "ok": false, "error": null, "completion_tokens": 2736, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/2", "ok": true, "error": null, "completion_tokens": 736, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/3", "ok": true, "error": null, "completion_tokens": 815, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/4", "ok": true, "error": null, "completion_tokens": 1225, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/5", "ok": false, "error": null, "completion_tokens": 588, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/6", "ok": true, "error": null, "completion_tokens": 827, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/7", "ok": false, "error": null, "completion_tokens": 423, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/8", "ok": true, "error": null, "completion_tokens": 549, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/9", "ok": false, "error": null, "completion_tokens": 485, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/10", "ok": true, "error": null, "completion_tokens": 13837, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/11", "ok": true, "error": null, "completion_tokens": 1578, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/12", "ok": false, "error": null, "completion_tokens": 754, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/13", "ok": true, "error": null, "completion_tokens": 461, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/14", "ok": true, "error": null, "completion_tokens": 1909, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/15", "ok": true, "error": null, "completion_tokens": 228, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/16", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/17", "ok": false, "error": null, "completion_tokens": 740, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/18", "ok": true, "error": null, "completion_tokens": 3133, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/19", "ok": true, "error": null, "completion_tokens": 781, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/20", "ok": false, "error": null, "completion_tokens": 1363, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/21", "ok": false, "error": null, "completion_tokens": 830, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/22", "ok": false, "error": null, "completion_tokens": 3780, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/23", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/24", "ok": true, "error": null, "completion_tokens": 3456, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/25", "ok": true, "error": null, "completion_tokens": 5281, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/26", "ok": false, "error": null, "completion_tokens": 782, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/27", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/28", "ok": false, "error": null, "completion_tokens": 487, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/29", "ok": false, "error": null, "completion_tokens": 662, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/30", "ok": true, "error": null, "completion_tokens": 259, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/31", "ok": true, "error": null, "completion_tokens": 1047, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/32", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/33", "ok": true, "error": null, "completion_tokens": 591, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/34", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/35", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/36", "ok": true, "error": null, "completion_tokens": 1240, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/37", "ok": true, "error": null, "completion_tokens": 594, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/38", "ok": false, "error": null, "completion_tokens": 747, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/39", "ok": true, "error": null, "completion_tokens": 5417, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/40", "ok": true, "error": null, "completion_tokens": 5025, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/41", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/42", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/43", "ok": true, "error": null, "completion_tokens": 497, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/44", "ok": true, "error": null, "completion_tokens": 1333, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/45", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/46", "ok": true, "error": null, "completion_tokens": 1753, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/47", "ok": false, "error": null, "completion_tokens": 19219, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/48", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/49", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/50", "ok": false, "error": null, "completion_tokens": 284, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/51", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/52", "ok": true, "error": null, "completion_tokens": 367, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/53", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/54", "ok": true, "error": null, "completion_tokens": 439, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/55", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/56", "ok": true, "error": null, "completion_tokens": 490, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/57", "ok": true, "error": null, "completion_tokens": 439, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/58", "ok": true, "error": null, "completion_tokens": 403, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/59", "ok": true, "error": null, "completion_tokens": 863, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/60", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/61", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/62", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/63", "ok": true, "error": null, "completion_tokens": 1065, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/64", "ok": true, "error": null, "completion_tokens": 689, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/65", "ok": true, "error": null, "completion_tokens": 1556, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/66", "ok": true, "error": null, "completion_tokens": 508, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/67", "ok": true, "error": null, "completion_tokens": 558, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/68", "ok": true, "error": null, "completion_tokens": 886, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/69", "ok": true, "error": null, "completion_tokens": 647, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/70", "ok": true, "error": null, "completion_tokens": 610, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/71", "ok": true, "error": null, "completion_tokens": 1948, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/72", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/73", "ok": true, "error": null, "completion_tokens": 603, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/74", "ok": true, "error": null, "completion_tokens": 568, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/75", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/76", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/77", "ok": true, "error": null, "completion_tokens": 2212, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/78", "ok": true, "error": null, "completion_tokens": 527, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/79", "ok": true, "error": null, "completion_tokens": 541, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/80", "ok": true, "error": null, "completion_tokens": 426, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/81", "ok": true, "error": null, "completion_tokens": 1368, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/82", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/83", "ok": true, "error": null, "completion_tokens": 1107, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/84", "ok": true, "error": null, "completion_tokens": 403, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/85", "ok": true, "error": null, "completion_tokens": 442, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/86", "ok": true, "error": null, "completion_tokens": 1238, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/87", "ok": true, "error": null, "completion_tokens": 747, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/88", "ok": true, "error": null, "completion_tokens": 742, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/89", "ok": true, "error": null, "completion_tokens": 924, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/90", "ok": true, "error": null, "completion_tokens": 816, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/91", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/92", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/93", "ok": true, "error": null, "completion_tokens": 3117, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/94", "ok": true, "error": null, "completion_tokens": 14504, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/95", "ok": true, "error": null, "completion_tokens": 1177, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/96", "ok": true, "error": null, "completion_tokens": 840, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/97", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/98", "ok": true, "error": null, "completion_tokens": 526, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/99", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/100", "ok": true, "error": null, "completion_tokens": 722, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/101", "ok": true, "error": null, "completion_tokens": 456, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/102", "ok": true, "error": null, "completion_tokens": 5762, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/103", "ok": true, "error": null, "completion_tokens": 2262, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/104", "ok": true, "error": null, "completion_tokens": 513, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/105", "ok": true, "error": null, "completion_tokens": 1176, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/106", "ok": true, "error": null, "completion_tokens": 1171, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/107", "ok": true, "error": null, "completion_tokens": 607, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/108", "ok": true, "error": null, "completion_tokens": 1211, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/109", "ok": true, "error": null, "completion_tokens": 2413, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/110", "ok": true, "error": null, "completion_tokens": 982, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/111", "ok": true, "error": null, "completion_tokens": 703, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/112", "ok": true, "error": null, "completion_tokens": 519, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/113", "ok": true, "error": null, "completion_tokens": 1733, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/114", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/115", "ok": true, "error": null, "completion_tokens": 975, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/116", "ok": false, "error": null, "completion_tokens": 22109, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/117", "ok": true, "error": null, "completion_tokens": 684, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/118", "ok": true, "error": null, "completion_tokens": 2237, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/119", "ok": true, "error": null, "completion_tokens": 794, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/120", "ok": true, "error": null, "completion_tokens": 560, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/121", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/122", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/123", "ok": true, "error": null, "completion_tokens": 915, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/124", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/125", "ok": true, "error": null, "completion_tokens": 3053, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/126", "ok": true, "error": null, "completion_tokens": 908, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/127", "ok": true, "error": null, "completion_tokens": 898, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/128", "ok": true, "error": null, "completion_tokens": 849, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/129", "ok": true, "error": null, "completion_tokens": 11639, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/130", "ok": true, "error": null, "completion_tokens": 2569, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/131", "ok": true, "error": null, "completion_tokens": 502, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/132", "ok": true, "error": null, "completion_tokens": 13778, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/133", "ok": true, "error": null, "completion_tokens": 1032, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/134", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/135", "ok": true, "error": null, "completion_tokens": 661, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/136", "ok": true, "error": null, "completion_tokens": 597, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/137", "ok": true, "error": null, "completion_tokens": 2440, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/138", "ok": true, "error": null, "completion_tokens": 451, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/139", "ok": true, "error": null, "completion_tokens": 462, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/140", "ok": true, "error": null, "completion_tokens": 762, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/141", "ok": true, "error": null, "completion_tokens": 1337, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/142", "ok": true, "error": null, "completion_tokens": 656, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/143", "ok": true, "error": null, "completion_tokens": 659, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/144", "ok": true, "error": null, "completion_tokens": 570, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/145", "ok": false, "error": null, "completion_tokens": 32463, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/146", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 0, "truncated": true }, { "id": "HumanEval/147", "ok": true, "error": null, "completion_tokens": 2809, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/148", "ok": true, "error": null, "completion_tokens": 758, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/149", "ok": true, "error": null, "completion_tokens": 965, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/150", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/151", "ok": true, "error": null, "completion_tokens": 6479, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/152", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/153", "ok": true, "error": null, "completion_tokens": 1417, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/154", "ok": true, "error": null, "completion_tokens": 736, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/155", "ok": true, "error": null, "completion_tokens": 487, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/156", "ok": true, "error": null, "completion_tokens": 561, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/157", "ok": true, "error": null, "completion_tokens": 4971, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/158", "ok": true, "error": null, "completion_tokens": 1031, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/159", "ok": true, "error": null, "completion_tokens": 466, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/160", "ok": true, "error": null, "completion_tokens": 2065, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/161", "ok": true, "error": null, "completion_tokens": 524, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/162", "ok": true, "error": null, "completion_tokens": 464, "reasoning_chars": 0, "truncated": false }, { "id": "HumanEval/163", "ok": true, "error": null, "completion_tokens": 2284, "reasoning_chars": 0, "truncated": false } ] }, "ifeval": { "task": "ifeval", "n": 100, "hits": 93, "accuracy_pct": 93.0, "accuracy_ex_trunc_pct": 93.0, "truncated": 0, "completion_tokens_p50": 1053, "completion_tokens_p90": 4032, "completion_tokens_max": 16872, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "errors": 0, "elapsed_s": 302.9, "gen_tokens": 183475, "tok_per_s": 605.8, "instruction_level_pct": 98.0, "instructions_checked": 150, "details": [ { "id": "ifeval-153", "ok": true, "error": null, "completion_tokens": 2013, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-258", "ok": true, "error": null, "completion_tokens": 436, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-368", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-85", "ok": true, "error": null, "completion_tokens": 1329, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-246", "ok": true, "error": null, "completion_tokens": 1399, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-23", "ok": true, "error": null, "completion_tokens": 2899, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-372", "ok": true, "error": null, "completion_tokens": 3022, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-387", "ok": true, "error": null, "completion_tokens": 848, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-138", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-56", "ok": true, "error": null, "completion_tokens": 1185, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-277", "ok": true, "error": null, "completion_tokens": 1208, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-407", "ok": false, "error": null, "completion_tokens": 1484, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-171", "ok": true, "error": null, "completion_tokens": 1755, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-96", "ok": true, "error": null, "completion_tokens": 4316, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-134", "ok": true, "error": null, "completion_tokens": 4195, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-218", "ok": true, "error": null, "completion_tokens": 467, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-330", "ok": true, "error": null, "completion_tokens": 3027, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-6", "ok": true, "error": null, "completion_tokens": 1334, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-143", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-121", "ok": true, "error": null, "completion_tokens": 837, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-219", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-524", "ok": true, "error": null, "completion_tokens": 387, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-206", "ok": true, "error": null, "completion_tokens": 535, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-144", "ok": true, "error": null, "completion_tokens": 1285, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-447", "ok": true, "error": null, "completion_tokens": 966, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-492", "ok": true, "error": null, "completion_tokens": 2542, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-485", "ok": true, "error": null, "completion_tokens": 16872, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-220", "ok": true, "error": null, "completion_tokens": 2617, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-412", "ok": true, "error": null, "completion_tokens": 3010, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-342", "ok": true, "error": null, "completion_tokens": 1605, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-222", "ok": true, "error": null, "completion_tokens": 8999, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-131", "ok": true, "error": null, "completion_tokens": 572, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-210", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-137", "ok": true, "error": null, "completion_tokens": 1179, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-464", "ok": true, "error": null, "completion_tokens": 754, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-503", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-446", "ok": true, "error": null, "completion_tokens": 792, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-2", "ok": true, "error": null, "completion_tokens": 1046, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-525", "ok": true, "error": null, "completion_tokens": 938, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-27", "ok": true, "error": null, "completion_tokens": 929, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-136", "ok": true, "error": null, "completion_tokens": 1668, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-286", "ok": true, "error": null, "completion_tokens": 1053, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-449", "ok": true, "error": null, "completion_tokens": 1217, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-55", "ok": true, "error": null, "completion_tokens": 7033, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-93", "ok": true, "error": null, "completion_tokens": 408, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-399", "ok": true, "error": null, "completion_tokens": 1626, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-19", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-229", "ok": true, "error": null, "completion_tokens": 948, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-181", "ok": true, "error": null, "completion_tokens": 507, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-362", "ok": true, "error": null, "completion_tokens": 731, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-289", "ok": true, "error": null, "completion_tokens": 380, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-356", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-115", "ok": false, "error": null, "completion_tokens": 2552, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-404", "ok": false, "error": null, "completion_tokens": 790, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-243", "ok": false, "error": null, "completion_tokens": 4281, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-90", "ok": true, "error": null, "completion_tokens": 2089, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-54", "ok": true, "error": null, "completion_tokens": 1341, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-276", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-308", "ok": true, "error": null, "completion_tokens": 1949, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-64", "ok": false, "error": null, "completion_tokens": 1078, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-260", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-337", "ok": true, "error": null, "completion_tokens": 1025, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-510", "ok": true, "error": null, "completion_tokens": 2930, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-119", "ok": true, "error": null, "completion_tokens": 1993, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-5", "ok": true, "error": null, "completion_tokens": 336, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-468", "ok": true, "error": null, "completion_tokens": 7435, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-418", "ok": true, "error": null, "completion_tokens": 869, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-476", "ok": true, "error": null, "completion_tokens": 3087, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-49", "ok": true, "error": null, "completion_tokens": 5697, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-336", "ok": true, "error": null, "completion_tokens": 283, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-422", "ok": true, "error": null, "completion_tokens": 1249, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-417", "ok": true, "error": null, "completion_tokens": 565, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-347", "ok": true, "error": null, "completion_tokens": 10154, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-101", "ok": true, "error": null, "completion_tokens": 554, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-209", "ok": true, "error": null, "completion_tokens": 522, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-341", "ok": true, "error": null, "completion_tokens": 4032, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-533", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-158", "ok": true, "error": null, "completion_tokens": 782, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-321", "ok": true, "error": null, "completion_tokens": 839, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-311", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-50", "ok": false, "error": null, "completion_tokens": 3485, "reasoning_chars": 0, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-518", "ok": true, "error": null, "completion_tokens": 3354, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-374", "ok": true, "error": null, "completion_tokens": 338, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-317", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-512", "ok": true, "error": null, "completion_tokens": 1811, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-102", "ok": true, "error": null, "completion_tokens": 619, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-72", "ok": true, "error": null, "completion_tokens": 1140, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-33", "ok": true, "error": null, "completion_tokens": 1275, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-263", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-501", "ok": true, "error": null, "completion_tokens": 3900, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-4", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-444", "ok": true, "error": null, "completion_tokens": 1287, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-228", "ok": true, "error": null, "completion_tokens": 794, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-299", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-278", "ok": true, "error": null, "completion_tokens": 5662, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-154", "ok": true, "error": null, "completion_tokens": 822, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-301", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-161", "ok": true, "error": null, "completion_tokens": 3710, "reasoning_chars": 0, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-394", "ok": false, "error": null, "completion_tokens": 1415, "reasoning_chars": 0, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-159", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 0, "truncated": false, "checked": 1, "passed": 1, "skipped": false } ] } } }, "original-w4a16": { "started": "2026-10-10 13:48:58", "finished": "2026-10-10 14:26:10", "seed": 20260816, "thinking": true, "tasks": { "mmlu": { "task": "mmlu", "n": 150, "hits": 126, "accuracy_pct": 84.0, "accuracy_ex_trunc_pct": 84.0, "truncated": 0, "completion_tokens_p50": 243, "completion_tokens_p90": 869, "completion_tokens_max": 26968, "reasoning_chars_p50": 824, "reasoning_chars_p90": 2898, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1346.1, "gen_tokens": 100578, "tok_per_s": 74.7, "details": [ { "id": "mmlu-12325", "ok": true, "error": null, "completion_tokens": 639, "reasoning_chars": 1213, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-154", "ok": false, "error": null, "completion_tokens": 220, "reasoning_chars": 569, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12844", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1153", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 549, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11089", "ok": true, "error": null, "completion_tokens": 915, "reasoning_chars": 2824, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12782", "ok": true, "error": null, "completion_tokens": 541, "reasoning_chars": 1798, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5422", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12363", "ok": true, "error": null, "completion_tokens": 923, "reasoning_chars": 2467, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12556", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13163", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-270", "ok": true, "error": null, "completion_tokens": 370, "reasoning_chars": 929, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6943", "ok": true, "error": null, "completion_tokens": 458, "reasoning_chars": 1950, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8330", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9276", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 331, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7584", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 322, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12819", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6163", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 793, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4099", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 824, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10389", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 252, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1539", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12013", "ok": true, "error": null, "completion_tokens": 572, "reasoning_chars": 1715, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7814", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 868, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1106", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1616", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 484, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-555", "ok": true, "error": null, "completion_tokens": 450, "reasoning_chars": 1115, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3698", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 1083, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-618", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 669, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9842", "ok": true, "error": null, "completion_tokens": 111, "reasoning_chars": 507, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4229", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 339, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1022", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 1050, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6421", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1997", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-8941", "ok": false, "error": null, "completion_tokens": 3824, "reasoning_chars": 17117, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-565", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 964, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11364", "ok": true, "error": null, "completion_tokens": 668, "reasoning_chars": 1100, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5959", "ok": true, "error": null, "completion_tokens": 588, "reasoning_chars": 1400, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13066", "ok": true, "error": null, "completion_tokens": 422, "reasoning_chars": 1760, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3122", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 433, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3722", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13839", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12570", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9959", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 418, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10791", "ok": true, "error": null, "completion_tokens": 1398, "reasoning_chars": 4971, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12268", "ok": true, "error": null, "completion_tokens": 785, "reasoning_chars": 2850, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9051", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 1132, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12887", "ok": true, "error": null, "completion_tokens": 638, "reasoning_chars": 2204, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2667", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 101, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3906", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 1149, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10304", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 527, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12927", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 286, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6337", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 498, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3907", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12888", "ok": false, "error": null, "completion_tokens": 260, "reasoning_chars": 1354, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10629", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 482, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5801", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 639, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2936", "ok": true, "error": null, "completion_tokens": 598, "reasoning_chars": 1549, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7389", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 1315, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9532", "ok": true, "error": null, "completion_tokens": 495, "reasoning_chars": 1663, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8705", "ok": true, "error": null, "completion_tokens": 465, "reasoning_chars": 2215, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11776", "ok": true, "error": null, "completion_tokens": 560, "reasoning_chars": 2451, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6836", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 883, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7856", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 509, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9697", "ok": true, "error": null, "completion_tokens": 537, "reasoning_chars": 1814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10825", "ok": true, "error": null, "completion_tokens": 1290, "reasoning_chars": 4714, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11178", "ok": true, "error": null, "completion_tokens": 708, "reasoning_chars": 3552, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11420", "ok": true, "error": null, "completion_tokens": 672, "reasoning_chars": 2898, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4658", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5403", "ok": false, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6160", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 1418, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6872", "ok": false, "error": null, "completion_tokens": 26968, "reasoning_chars": 115278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9738", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 1139, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3076", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13496", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 1022, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11289", "ok": false, "error": null, "completion_tokens": 116, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-222", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 1142, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12180", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1143", "ok": false, "error": null, "completion_tokens": 48, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13086", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7038", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13587", "ok": true, "error": null, "completion_tokens": 666, "reasoning_chars": 3602, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12349", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 467, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11747", "ok": true, "error": null, "completion_tokens": 539, "reasoning_chars": 1547, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9848", "ok": false, "error": null, "completion_tokens": 914, "reasoning_chars": 3251, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3438", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 827, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-89", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 1008, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4253", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 314, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5460", "ok": true, "error": null, "completion_tokens": 790, "reasoning_chars": 2504, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2494", "ok": false, "error": null, "completion_tokens": 9645, "reasoning_chars": 31437, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11149", "ok": false, "error": null, "completion_tokens": 1714, "reasoning_chars": 6082, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-718", "ok": true, "error": null, "completion_tokens": 412, "reasoning_chars": 1182, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5330", "ok": false, "error": null, "completion_tokens": 441, "reasoning_chars": 1374, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10716", "ok": true, "error": null, "completion_tokens": 4170, "reasoning_chars": 16135, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5636", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 577, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11832", "ok": true, "error": null, "completion_tokens": 646, "reasoning_chars": 1515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-95", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 1198, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13960", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 598, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2072", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 17, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12030", "ok": true, "error": null, "completion_tokens": 1356, "reasoning_chars": 5442, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2931", "ok": false, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2441", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 294, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13666", "ok": false, "error": null, "completion_tokens": 280, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6575", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 562, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11625", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13283", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-4262", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 581, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12931", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12150", "ok": true, "error": null, "completion_tokens": 1015, "reasoning_chars": 3242, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3678", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 1041, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4593", "ok": true, "error": null, "completion_tokens": 561, "reasoning_chars": 1535, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1563", "ok": true, "error": null, "completion_tokens": 672, "reasoning_chars": 3089, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13396", "ok": true, "error": null, "completion_tokens": 502, "reasoning_chars": 1332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8049", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 740, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8396", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 1588, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8493", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 1001, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13514", "ok": true, "error": null, "completion_tokens": 416, "reasoning_chars": 1624, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3000", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 120, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1635", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 514, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1391", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 543, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-627", "ok": false, "error": null, "completion_tokens": 10126, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1573", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 813, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6731", "ok": true, "error": null, "completion_tokens": 86, "reasoning_chars": 352, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11665", "ok": true, "error": null, "completion_tokens": 799, "reasoning_chars": 3821, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2914", "ok": true, "error": null, "completion_tokens": 467, "reasoning_chars": 1599, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6875", "ok": true, "error": null, "completion_tokens": 1184, "reasoning_chars": 5395, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2184", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 651, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12801", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 1432, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6717", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 529, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 639, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12464", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2787", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 1111, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11549", "ok": true, "error": null, "completion_tokens": 517, "reasoning_chars": 1576, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-842", "ok": true, "error": null, "completion_tokens": 869, "reasoning_chars": 2466, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13868", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9647", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 954, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9502", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12441", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 1380, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4392", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 292, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5036", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 672, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12606", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 1362, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1400", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 1083, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13979", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13634", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 897, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11226", "ok": true, "error": null, "completion_tokens": 880, "reasoning_chars": 2830, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-625", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-257", "ok": true, "error": null, "completion_tokens": 228, "reasoning_chars": 848, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12853", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 642, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-451", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 1329, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2074", "ok": true, "error": null, "completion_tokens": 56, "reasoning_chars": 138, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9117", "ok": true, "error": null, "completion_tokens": 376, "reasoning_chars": 1272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10748", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "cmmlu": { "task": "cmmlu", "n": 150, "hits": 127, "accuracy_pct": 84.67, "accuracy_ex_trunc_pct": 84.67, "truncated": 0, "completion_tokens_p50": 210, "completion_tokens_p90": 933, "completion_tokens_max": 19215, "reasoning_chars_p50": 386, "reasoning_chars_p90": 2015, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1397.6, "gen_tokens": 82459, "tok_per_s": 59.0, "details": [ { "id": "cmmlu-chinese_teacher_qualification-63", "ok": true, "error": null, "completion_tokens": 230, "reasoning_chars": 366, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-54", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-91", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 59, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-260", "ok": true, "error": null, "completion_tokens": 498, "reasoning_chars": 2080, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-32", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 225, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-29", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 392, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-200", "ok": false, "error": null, "completion_tokens": 2118, "reasoning_chars": 3220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-98", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 393, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-44", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-118", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 565, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-116", "ok": true, "error": null, "completion_tokens": 1191, "reasoning_chars": 1667, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-43", "ok": true, "error": null, "completion_tokens": 490, "reasoning_chars": 669, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-145", "ok": true, "error": null, "completion_tokens": 516, "reasoning_chars": 2045, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-247", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 354, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-224", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 148, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-26", "ok": true, "error": null, "completion_tokens": 724, "reasoning_chars": 1547, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-105", "ok": true, "error": null, "completion_tokens": 989, "reasoning_chars": 1277, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-15", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 920, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-62", "ok": false, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-nutrition-74", "ok": true, "error": null, "completion_tokens": 178, "reasoning_chars": 303, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-181", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 204, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-127", "ok": true, "error": null, "completion_tokens": 503, "reasoning_chars": 885, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_geography-107", "ok": false, "error": null, "completion_tokens": 349, "reasoning_chars": 634, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-18", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 598, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-82", "ok": true, "error": null, "completion_tokens": 116, "reasoning_chars": 499, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_food_culture-38", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 149, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-198", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-96", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 95, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-73", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-20", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-95", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 617, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-88", "ok": true, "error": null, "completion_tokens": 1497, "reasoning_chars": 6607, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-192", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 477, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-66", "ok": true, "error": null, "completion_tokens": 395, "reasoning_chars": 420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-100", "ok": true, "error": null, "completion_tokens": 310, "reasoning_chars": 1288, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-136", "ok": true, "error": null, "completion_tokens": 236, "reasoning_chars": 947, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-16", "ok": false, "error": null, "completion_tokens": 304, "reasoning_chars": 1447, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-90", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 1093, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-41", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-345", "ok": true, "error": null, "completion_tokens": 439, "reasoning_chars": 1564, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-103", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-93", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-88", "ok": true, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-206", "ok": true, "error": null, "completion_tokens": 254, "reasoning_chars": 408, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-105", "ok": true, "error": null, "completion_tokens": 686, "reasoning_chars": 785, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-90", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 816, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-190", "ok": false, "error": null, "completion_tokens": 571, "reasoning_chars": 735, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-international_law-39", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 442, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-155", "ok": true, "error": null, "completion_tokens": 684, "reasoning_chars": 1144, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-24", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 335, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-6", "ok": true, "error": null, "completion_tokens": 509, "reasoning_chars": 1835, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-31", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-243", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 647, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-63", "ok": false, "error": null, "completion_tokens": 1263, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-135", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 29, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-89", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 31, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-91", "ok": false, "error": null, "completion_tokens": 659, "reasoning_chars": 790, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-120", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 432, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-62", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-106", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 154, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-food_science-106", "ok": false, "error": null, "completion_tokens": 216, "reasoning_chars": 379, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-24", "ok": true, "error": null, "completion_tokens": 679, "reasoning_chars": 1164, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-157", "ok": true, "error": null, "completion_tokens": 567, "reasoning_chars": 729, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-41", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 1526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-115", "ok": true, "error": null, "completion_tokens": 2004, "reasoning_chars": 6243, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-journalism-29", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 142, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-67", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 129, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-161", "ok": true, "error": null, "completion_tokens": 65, "reasoning_chars": 65, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-120", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 1176, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-education-117", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 212, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-169", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 280, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-129", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-19", "ok": true, "error": null, "completion_tokens": 1356, "reasoning_chars": 5096, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-8", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 648, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-72", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-139", "ok": false, "error": null, "completion_tokens": 646, "reasoning_chars": 1087, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_security-22", "ok": true, "error": null, "completion_tokens": 205, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-7", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 85, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-108", "ok": true, "error": null, "completion_tokens": 1218, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-31", "ok": true, "error": null, "completion_tokens": 26, "reasoning_chars": 28, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-131", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 446, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-35", "ok": false, "error": null, "completion_tokens": 45, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_actuarial_science-38", "ok": true, "error": null, "completion_tokens": 1903, "reasoning_chars": 3614, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-165", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 368, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-101", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 677, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-107", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 129, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-8", "ok": false, "error": null, "completion_tokens": 6143, "reasoning_chars": 25187, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medical_statistics-86", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 837, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-security_study-30", "ok": true, "error": null, "completion_tokens": 314, "reasoning_chars": 520, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-118", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 934, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_law-24", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-46", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 626, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-anatomy-21", "ok": false, "error": null, "completion_tokens": 486, "reasoning_chars": 732, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-145", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 379, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-91", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 30, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-109", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 11, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-44", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-74", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 126, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-63", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 195, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-146", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-109", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 437, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-construction_project_management-16", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 174, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-113", "ok": false, "error": null, "completion_tokens": 153, "reasoning_chars": 262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-102", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 491, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-207", "ok": false, "error": null, "completion_tokens": 972, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-19", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 196, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-164", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 293, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-338", "ok": true, "error": null, "completion_tokens": 933, "reasoning_chars": 2729, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-7", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 1247, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-92", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 761, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-47", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 651, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-140", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 71, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-100", "ok": true, "error": null, "completion_tokens": 101, "reasoning_chars": 147, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-37", "ok": true, "error": null, "completion_tokens": 764, "reasoning_chars": 2015, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-191", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 60, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-34", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-88", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 397, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-77", "ok": false, "error": null, "completion_tokens": 340, "reasoning_chars": 1284, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-306", "ok": false, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-40", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 366, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_chemistry-7", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-152", "ok": true, "error": null, "completion_tokens": 423, "reasoning_chars": 785, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-1", "ok": true, "error": null, "completion_tokens": 42, "reasoning_chars": 53, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-94", "ok": true, "error": null, "completion_tokens": 86, "reasoning_chars": 128, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-109", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 379, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-164", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 94, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-89", "ok": true, "error": null, "completion_tokens": 804, "reasoning_chars": 3134, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-17", "ok": true, "error": null, "completion_tokens": 599, "reasoning_chars": 2188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-2", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 183, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-51", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 943, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-141", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 1411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-196", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 69, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-97", "ok": true, "error": null, "completion_tokens": 40, "reasoning_chars": 42, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-202", "ok": true, "error": null, "completion_tokens": 8714, "reasoning_chars": 35180, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-195", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 264, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-49", "ok": true, "error": null, "completion_tokens": 1434, "reasoning_chars": 2802, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-94", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-8", "ok": false, "error": null, "completion_tokens": 556, "reasoning_chars": 734, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-98", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 233, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-28", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 466, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-74", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-121", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 409, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-120", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 387, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-105", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 85, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-35", "ok": true, "error": null, "completion_tokens": 19215, "reasoning_chars": 52471, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-51", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 308, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-49", "ok": true, "error": null, "completion_tokens": 1675, "reasoning_chars": 4077, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-130", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 1460, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-140", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 867, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "mmlu_pro": { "task": "mmlu_pro", "n": 150, "hits": 118, "accuracy_pct": 78.67, "accuracy_ex_trunc_pct": 80.27, "truncated": 3, "completion_tokens_p50": 271, "completion_tokens_p90": 2190, "completion_tokens_max": 32768, "reasoning_chars_p50": 759, "reasoning_chars_p90": 4751, "reasoning_leaked": null, "empty": 3, "length_limited": 3, "errors": 0, "elapsed_s": 1945.5, "gen_tokens": 274989, "tok_per_s": 141.3, "details": [ { "id": "mmlupro-5831", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 773, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2299", "ok": true, "error": null, "completion_tokens": 493, "reasoning_chars": 1757, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9280", "ok": true, "error": null, "completion_tokens": 2168, "reasoning_chars": 6332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5486", "ok": true, "error": null, "completion_tokens": 643, "reasoning_chars": 2819, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5761", "ok": true, "error": null, "completion_tokens": 528, "reasoning_chars": 1360, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6589", "ok": true, "error": null, "completion_tokens": 834, "reasoning_chars": 2378, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5389", "ok": true, "error": null, "completion_tokens": 254, "reasoning_chars": 737, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5131", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1677", "ok": true, "error": null, "completion_tokens": 1594, "reasoning_chars": 5852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4388", "ok": false, "error": null, "completion_tokens": 11381, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3285", "ok": true, "error": null, "completion_tokens": 1080, "reasoning_chars": 2989, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4118", "ok": true, "error": null, "completion_tokens": 1277, "reasoning_chars": 3665, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5344", "ok": false, "error": null, "completion_tokens": 63, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11696", "ok": true, "error": null, "completion_tokens": 3318, "reasoning_chars": 11349, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-858", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 305, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-214", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 103617, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlupro-6609", "ok": true, "error": null, "completion_tokens": 920, "reasoning_chars": 2411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2083", "ok": true, "error": null, "completion_tokens": 667, "reasoning_chars": 2852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9752", "ok": false, "error": null, "completion_tokens": 21877, "reasoning_chars": 66860, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7213", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5234", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 378, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8570", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 363, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5612", "ok": true, "error": null, "completion_tokens": 111, "reasoning_chars": 167, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2071", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9842", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4665", "ok": true, "error": null, "completion_tokens": 1209, "reasoning_chars": 2325, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5564", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6475", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2153", "ok": true, "error": null, "completion_tokens": 367, "reasoning_chars": 1239, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5812", "ok": true, "error": null, "completion_tokens": 137, "reasoning_chars": 231, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3407", "ok": true, "error": null, "completion_tokens": 3328, "reasoning_chars": 3338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2879", "ok": false, "error": null, "completion_tokens": 1366, "reasoning_chars": 4751, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8266", "ok": false, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1916", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3486", "ok": false, "error": null, "completion_tokens": 730, "reasoning_chars": 2060, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11275", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6040", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8565", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 354, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6848", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8926", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8458", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7854", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 951, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3962", "ok": true, "error": null, "completion_tokens": 1344, "reasoning_chars": 2786, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6470", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3813", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 788, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8026", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 201, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11030", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8151", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 15, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5286", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3888", "ok": true, "error": null, "completion_tokens": 40, "reasoning_chars": 127, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11676", "ok": true, "error": null, "completion_tokens": 2835, "reasoning_chars": 5659, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8762", "ok": false, "error": null, "completion_tokens": 1026, "reasoning_chars": 1912, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-12160", "ok": true, "error": null, "completion_tokens": 548, "reasoning_chars": 1403, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4022", "ok": true, "error": null, "completion_tokens": 2507, "reasoning_chars": 5832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10092", "ok": true, "error": null, "completion_tokens": 502, "reasoning_chars": 1658, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7364", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 939, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6001", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10873", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9777", "ok": true, "error": null, "completion_tokens": 2190, "reasoning_chars": 4370, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8723", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 476, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4595", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 762, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8423", "ok": true, "error": null, "completion_tokens": 5591, "reasoning_chars": 9333, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11051", "ok": false, "error": null, "completion_tokens": 193, "reasoning_chars": 950, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-836", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 140, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11297", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 98267, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlupro-3950", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 511, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1562", "ok": false, "error": null, "completion_tokens": 1125, "reasoning_chars": 4165, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2041", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 2698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3473", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 231, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11253", "ok": true, "error": null, "completion_tokens": 2027, "reasoning_chars": 8852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3858", "ok": true, "error": null, "completion_tokens": 789, "reasoning_chars": 1627, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2231", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2190", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 1143, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6930", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 1790, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6558", "ok": true, "error": null, "completion_tokens": 718, "reasoning_chars": 2019, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5371", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1766", "ok": true, "error": null, "completion_tokens": 945, "reasoning_chars": 3207, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6045", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2298", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6400", "ok": true, "error": null, "completion_tokens": 208, "reasoning_chars": 839, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6499", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7936", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 1531, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8566", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 247, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11633", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 1179, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9843", "ok": false, "error": null, "completion_tokens": 10732, "reasoning_chars": 26003, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11758", "ok": false, "error": null, "completion_tokens": 19857, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2843", "ok": true, "error": null, "completion_tokens": 1214, "reasoning_chars": 3219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7483", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11379", "ok": true, "error": null, "completion_tokens": 1580, "reasoning_chars": 4275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4866", "ok": true, "error": null, "completion_tokens": 775, "reasoning_chars": 2382, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-818", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2785", "ok": true, "error": null, "completion_tokens": 943, "reasoning_chars": 4093, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8119", "ok": true, "error": null, "completion_tokens": 487, "reasoning_chars": 1648, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2214", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5774", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6049", "ok": false, "error": null, "completion_tokens": 526, "reasoning_chars": 1441, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7219", "ok": true, "error": null, "completion_tokens": 438, "reasoning_chars": 1548, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1660", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4767", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5964", "ok": false, "error": null, "completion_tokens": 414, "reasoning_chars": 1122, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2743", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 1198, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4062", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 680, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7368", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 638, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-827", "ok": false, "error": null, "completion_tokens": 503, "reasoning_chars": 2021, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11877", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 71786, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlupro-7557", "ok": true, "error": null, "completion_tokens": 554, "reasoning_chars": 1624, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9998", "ok": false, "error": null, "completion_tokens": 86, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10098", "ok": false, "error": null, "completion_tokens": 914, "reasoning_chars": 3789, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1744", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9210", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 460, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11248", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 741, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11187", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 125, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2799", "ok": true, "error": null, "completion_tokens": 283, "reasoning_chars": 1321, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4479", "ok": true, "error": null, "completion_tokens": 2903, "reasoning_chars": 6282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10223", "ok": true, "error": null, "completion_tokens": 634, "reasoning_chars": 1257, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5698", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 524, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8291", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 353, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3516", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5466", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4851", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-733", "ok": true, "error": null, "completion_tokens": 1197, "reasoning_chars": 1597, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6995", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 468, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2573", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8481", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 655, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5938", "ok": true, "error": null, "completion_tokens": 756, "reasoning_chars": 2804, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10708", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8476", "ok": true, "error": null, "completion_tokens": 364, "reasoning_chars": 879, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7990", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 748, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2409", "ok": true, "error": null, "completion_tokens": 510, "reasoning_chars": 2456, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11551", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-674", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 270, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1978", "ok": true, "error": null, "completion_tokens": 973, "reasoning_chars": 3227, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10420", "ok": false, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3445", "ok": true, "error": null, "completion_tokens": 838, "reasoning_chars": 2751, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9488", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4993", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 446, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4316", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 267, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4730", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11570", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 903, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2149", "ok": false, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8322", "ok": true, "error": null, "completion_tokens": 1093, "reasoning_chars": 1905, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10993", "ok": false, "error": null, "completion_tokens": 503, "reasoning_chars": 1922, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4588", "ok": true, "error": null, "completion_tokens": 1594, "reasoning_chars": 2938, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3258", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8629", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-637", "ok": true, "error": null, "completion_tokens": 314, "reasoning_chars": 724, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4257", "ok": true, "error": null, "completion_tokens": 275, "reasoning_chars": 759, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9719", "ok": true, "error": null, "completion_tokens": 1348, "reasoning_chars": 3507, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11714", "ok": true, "error": null, "completion_tokens": 24837, "reasoning_chars": 57879, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9678", "ok": false, "error": null, "completion_tokens": 12445, "reasoning_chars": 57196, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ceval": { "task": "ceval", "n": 150, "hits": 126, "accuracy_pct": 84.0, "accuracy_ex_trunc_pct": 84.56, "truncated": 1, "completion_tokens_p50": 240, "completion_tokens_p90": 1291, "completion_tokens_max": 32768, "reasoning_chars_p50": 390, "reasoning_chars_p90": 3986, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 1484.2, "gen_tokens": 193631, "tok_per_s": 130.5, "details": [ { "id": "ceval-high_school_chinese-1", "ok": false, "error": null, "completion_tokens": 4988, "reasoning_chars": 15262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-10", "ok": true, "error": null, "completion_tokens": 537, "reasoning_chars": 797, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-41", "ok": false, "error": null, "completion_tokens": 6145, "reasoning_chars": 18133, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-3", "ok": true, "error": null, "completion_tokens": 283, "reasoning_chars": 1185, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-21", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 274, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-4", "ok": true, "error": null, "completion_tokens": 1898, "reasoning_chars": 4277, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-21", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 288, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-2", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 90, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-6", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-12", "ok": true, "error": null, "completion_tokens": 7155, "reasoning_chars": 23431, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-19", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 198, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chemistry-9", "ok": true, "error": null, "completion_tokens": 645, "reasoning_chars": 2268, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-15", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-22", "ok": false, "error": null, "completion_tokens": 354, "reasoning_chars": 596, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-16", "ok": true, "error": null, "completion_tokens": 40, "reasoning_chars": 51, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-11", "ok": true, "error": null, "completion_tokens": 2876, "reasoning_chars": 4472, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-18", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 152, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-24", "ok": true, "error": null, "completion_tokens": 515, "reasoning_chars": 1481, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-29", "ok": true, "error": null, "completion_tokens": 1389, "reasoning_chars": 5697, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-2", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 264, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-11", "ok": true, "error": null, "completion_tokens": 119, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-logic-18", "ok": true, "error": null, "completion_tokens": 493, "reasoning_chars": 1989, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-0", "ok": false, "error": null, "completion_tokens": 892, "reasoning_chars": 2842, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-14", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-27", "ok": true, "error": null, "completion_tokens": 1225, "reasoning_chars": 4144, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-19", "ok": false, "error": null, "completion_tokens": 895, "reasoning_chars": 1165, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-21", "ok": true, "error": null, "completion_tokens": 4073, "reasoning_chars": 16554, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-33", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 249, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-20", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 1204, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-10", "ok": true, "error": null, "completion_tokens": 532, "reasoning_chars": 2474, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-19", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 245, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-7", "ok": true, "error": null, "completion_tokens": 116, "reasoning_chars": 174, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-41", "ok": true, "error": null, "completion_tokens": 639, "reasoning_chars": 1946, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-44", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 320, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-9", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 445, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-28", "ok": true, "error": null, "completion_tokens": 277, "reasoning_chars": 1380, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-39", "ok": true, "error": null, "completion_tokens": 792, "reasoning_chars": 2528, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-1", "ok": false, "error": null, "completion_tokens": 304, "reasoning_chars": 1058, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-24", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-5", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-computer_architecture-15", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-39", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-law-23", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 283, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-7", "ok": false, "error": null, "completion_tokens": 228, "reasoning_chars": 359, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-17", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 368, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-6", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-15", "ok": true, "error": null, "completion_tokens": 261, "reasoning_chars": 389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-4", "ok": true, "error": null, "completion_tokens": 12335, "reasoning_chars": 34986, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-5", "ok": true, "error": null, "completion_tokens": 723, "reasoning_chars": 2832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-50", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 387, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_politics-11", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 148, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-32", "ok": true, "error": null, "completion_tokens": 595, "reasoning_chars": 2095, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-16", "ok": true, "error": null, "completion_tokens": 357, "reasoning_chars": 1086, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-0", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 455, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-9", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-15", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 440, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-2", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 60192, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "ceval-fire_engineer-7", "ok": true, "error": null, "completion_tokens": 1189, "reasoning_chars": 1728, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-11", "ok": true, "error": null, "completion_tokens": 367, "reasoning_chars": 1363, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-operating_system-18", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 142, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-4", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-17", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-1", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-32", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 110, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-33", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-25", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 359, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-40", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-7", "ok": true, "error": null, "completion_tokens": 1291, "reasoning_chars": 4170, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-5", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-12", "ok": true, "error": null, "completion_tokens": 192, "reasoning_chars": 347, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-4", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 214, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-38", "ok": false, "error": null, "completion_tokens": 28756, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-22", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-15", "ok": false, "error": null, "completion_tokens": 18399, "reasoning_chars": 44674, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-45", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 60, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-6", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 868, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-20", "ok": true, "error": null, "completion_tokens": 499, "reasoning_chars": 722, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-2", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 448, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-17", "ok": true, "error": null, "completion_tokens": 966, "reasoning_chars": 3483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-16", "ok": false, "error": null, "completion_tokens": 628, "reasoning_chars": 2574, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-7", "ok": true, "error": null, "completion_tokens": 269, "reasoning_chars": 695, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_biology-16", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-10", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 772, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-32", "ok": true, "error": null, "completion_tokens": 3243, "reasoning_chars": 8911, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-0", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 112, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-12", "ok": true, "error": null, "completion_tokens": 1248, "reasoning_chars": 4282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-14", "ok": true, "error": null, "completion_tokens": 473, "reasoning_chars": 2272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-26", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 456, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-3", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 16, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-3", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 620, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-13", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 198, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-5", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 309, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-27", "ok": true, "error": null, "completion_tokens": 769, "reasoning_chars": 3089, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-21", "ok": false, "error": null, "completion_tokens": 17476, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-10", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 137, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-16", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-49", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 1100, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-13", "ok": false, "error": null, "completion_tokens": 344, "reasoning_chars": 1275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-5", "ok": true, "error": null, "completion_tokens": 7158, "reasoning_chars": 22946, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-2", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 74, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-27", "ok": false, "error": null, "completion_tokens": 919, "reasoning_chars": 1659, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_biology-8", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-14", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-14", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 268, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-31", "ok": true, "error": null, "completion_tokens": 555, "reasoning_chars": 919, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-13", "ok": true, "error": null, "completion_tokens": 434, "reasoning_chars": 1503, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-12", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 830, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-44", "ok": false, "error": null, "completion_tokens": 625, "reasoning_chars": 642, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-3", "ok": true, "error": null, "completion_tokens": 1036, "reasoning_chars": 2740, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-13", "ok": true, "error": null, "completion_tokens": 6007, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-6", "ok": true, "error": null, "completion_tokens": 86, "reasoning_chars": 203, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_politics-19", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 826, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-environmental_impact_assessment_engineer-27", "ok": false, "error": null, "completion_tokens": 595, "reasoning_chars": 873, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-10", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 293, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-3", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 295, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-6", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 393, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chinese-13", "ok": true, "error": null, "completion_tokens": 298, "reasoning_chars": 416, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-43", "ok": false, "error": null, "completion_tokens": 263, "reasoning_chars": 390, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-42", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 23, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-13", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 1215, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-discrete_mathematics-1", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 194, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-11", "ok": true, "error": null, "completion_tokens": 579, "reasoning_chars": 2510, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-16", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 269, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-2", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-1", "ok": true, "error": null, "completion_tokens": 422, "reasoning_chars": 957, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-16", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-11", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 314, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-5", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 418, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-40", "ok": true, "error": null, "completion_tokens": 230, "reasoning_chars": 1129, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-6", "ok": false, "error": null, "completion_tokens": 363, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-10", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 229, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-10", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-17", "ok": false, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-11", "ok": true, "error": null, "completion_tokens": 1113, "reasoning_chars": 3986, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-16", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-1", "ok": false, "error": null, "completion_tokens": 292, "reasoning_chars": 473, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-0", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 410, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-37", "ok": true, "error": null, "completion_tokens": 116, "reasoning_chars": 193, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-9", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 250, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_history-16", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 391, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-5", "ok": true, "error": null, "completion_tokens": 27, "reasoning_chars": 26, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-18", "ok": true, "error": null, "completion_tokens": 485, "reasoning_chars": 1170, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-30", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 194, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-21", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 423, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-42", "ok": true, "error": null, "completion_tokens": 360, "reasoning_chars": 1199, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-5", "ok": true, "error": null, "completion_tokens": 801, "reasoning_chars": 2872, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-4", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "arc": { "task": "arc", "n": 150, "hits": 121, "accuracy_pct": 80.67, "accuracy_ex_trunc_pct": 80.67, "truncated": 0, "completion_tokens_p50": 135, "completion_tokens_p90": 437, "completion_tokens_max": 1740, "reasoning_chars_p50": 519, "reasoning_chars_p90": 1322, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1383.3, "gen_tokens": 30534, "tok_per_s": 22.1, "details": [ { "id": "arc-Mercury_415084", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 231, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7249708", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 1322, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_4_24", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7056823", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 721, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415534", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 461, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405827", "ok": false, "error": null, "completion_tokens": 1740, "reasoning_chars": 7826, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263183", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 509, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106960", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 190, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_7", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 561, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7182245", "ok": true, "error": null, "completion_tokens": 485, "reasoning_chars": 1779, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416177", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2011_5_8", "ok": true, "error": null, "completion_tokens": 336, "reasoning_chars": 948, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_8_36", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 140, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401699", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 317, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107240", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 207, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_4_26", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 473, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402638", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026723", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415422", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 1010, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2000_8_2", "ok": true, "error": null, "completion_tokens": 557, "reasoning_chars": 1264, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2006_9_4", "ok": true, "error": null, "completion_tokens": 283, "reasoning_chars": 1071, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401272", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416169", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 372, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7116235", "ok": true, "error": null, "completion_tokens": 437, "reasoning_chars": 1263, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7109463", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 809, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7032428", "ok": true, "error": null, "completion_tokens": 683, "reasoning_chars": 2189, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2008_8_11", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 470, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_410334", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 1098, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_36", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415719", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 305, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_17", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 1073, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402256", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 366, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_400443", "ok": true, "error": null, "completion_tokens": 101, "reasoning_chars": 338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7133245", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 1039, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413458", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401372", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 820, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401116", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__5_10312", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 415, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2007_4_10280", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 271, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416683", "ok": true, "error": null, "completion_tokens": 544, "reasoning_chars": 1375, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7190120", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 579, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401295", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 261, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_4_28", "ok": true, "error": null, "completion_tokens": 48, "reasoning_chars": 197, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7223493", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 519, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018410", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 525, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1998_8_24", "ok": true, "error": null, "completion_tokens": 692, "reasoning_chars": 1795, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_31", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 925, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405501", "ok": true, "error": null, "completion_tokens": 277, "reasoning_chars": 994, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408030", "ok": true, "error": null, "completion_tokens": 340, "reasoning_chars": 854, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401684", "ok": false, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041668", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7205870", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 843, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7210140", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 1168, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405873", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 1187, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400052", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018270", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 1326, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406550", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 800, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7150728", "ok": true, "error": null, "completion_tokens": 277, "reasoning_chars": 1261, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400839", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_30", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400383", "ok": true, "error": null, "completion_tokens": 495, "reasoning_chars": 1351, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7236740", "ok": true, "error": null, "completion_tokens": 212, "reasoning_chars": 733, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-CSZ_2005_5_CSZ10021", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 465, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7093958", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 1025, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026355", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408324", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 432, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__7_10348", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402629", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 258, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400046", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 977, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7180810", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 1378, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_180005", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2009_8_14", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 1222, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2007_5_16", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_LBS10662", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 607, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_23", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 445, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2012_8_19", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041650", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 808, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416181", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 410, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_1995_8_J6", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 978, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2014_5_13", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 943, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106750", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 541, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_4_23", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 760, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MEA_2013_5_12", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 228, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_8_42", "ok": false, "error": null, "completion_tokens": 176, "reasoning_chars": 460, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408367", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 417, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7250058", "ok": true, "error": null, "completion_tokens": 393, "reasoning_chars": 940, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_414041", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 969, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401290", "ok": true, "error": null, "completion_tokens": 395, "reasoning_chars": 1038, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409295", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 477, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7211680", "ok": true, "error": null, "completion_tokens": 447, "reasoning_chars": 1399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_37", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 1114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413439", "ok": true, "error": null, "completion_tokens": 394, "reasoning_chars": 1210, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_32", "ok": true, "error": null, "completion_tokens": 51, "reasoning_chars": 205, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7216423", "ok": true, "error": null, "completion_tokens": 343, "reasoning_chars": 1055, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2015_7_9", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7230405", "ok": true, "error": null, "completion_tokens": 513, "reasoning_chars": 1460, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7025008", "ok": true, "error": null, "completion_tokens": 489, "reasoning_chars": 1263, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416536", "ok": true, "error": null, "completion_tokens": 76, "reasoning_chars": 336, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_2007_8_pg60", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 556, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_4_34", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 1334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_406833", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 953, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409647", "ok": true, "error": null, "completion_tokens": 466, "reasoning_chars": 1398, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_184363", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107170", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 707, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416104", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_10-v1", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7202843", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029803", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 478, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7164623", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 897, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_26", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263060", "ok": true, "error": null, "completion_tokens": 440, "reasoning_chars": 1189, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268328", "ok": true, "error": null, "completion_tokens": 102, "reasoning_chars": 424, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402282", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 720, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029558", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 144, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_4_21", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7186358", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 1173, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2011_8_10434", "ok": true, "error": null, "completion_tokens": 437, "reasoning_chars": 1214, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2012_8_23651", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 793, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_407125", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 648, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268258", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7027405", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 484, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_8_7", "ok": true, "error": null, "completion_tokens": 495, "reasoning_chars": 2375, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_191625", "ok": false, "error": null, "completion_tokens": 405, "reasoning_chars": 1363, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_6", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 216, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7219678", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 849, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401678", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 748, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401312", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_8_42", "ok": false, "error": null, "completion_tokens": 89, "reasoning_chars": 184, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7030468", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 315, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175385", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175560", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_5", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 376, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_409009", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7128923", "ok": false, "error": null, "completion_tokens": 332, "reasoning_chars": 1482, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_8_9", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7042945", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 1163, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_8_47", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 689, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7185343", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 980, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7218400", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 499, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7221988", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405950", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 761, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7222530", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 1188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7102323", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_5_39", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 430, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_8_5", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 952, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400661", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7081603", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 647, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2008_3_25", "ok": false, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413628", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406427", "ok": true, "error": null, "completion_tokens": 465, "reasoning_chars": 1151, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "truthfulqa": { "task": "truthfulqa", "n": 150, "hits": 88, "accuracy_pct": 58.67, "accuracy_ex_trunc_pct": 58.67, "truncated": 0, "completion_tokens_p50": 27, "completion_tokens_p90": 445, "completion_tokens_max": 980, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1322, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1402.0, "gen_tokens": 22891, "tok_per_s": 16.3, "details": [ { "id": "tqa-210", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-76", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-461", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-218", "ok": true, "error": null, "completion_tokens": 124, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-254", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-635", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 861, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-193", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 75, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-148", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-400", "ok": false, "error": null, "completion_tokens": 388, "reasoning_chars": 1243, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-766", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-240", "ok": false, "error": null, "completion_tokens": 898, "reasoning_chars": 3291, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-50", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-83", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-381", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-325", "ok": false, "error": null, "completion_tokens": 256, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-655", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-721", "ok": true, "error": null, "completion_tokens": 364, "reasoning_chars": 977, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-391", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-401", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 1188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-625", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 941, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-80", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-439", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-66", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-155", "ok": true, "error": null, "completion_tokens": 445, "reasoning_chars": 1109, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-516", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-522", "ok": false, "error": null, "completion_tokens": 164, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-47", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 1593, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-343", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-126", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-77", "ok": false, "error": null, "completion_tokens": 63, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-613", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 98, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-590", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 902, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-674", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-438", "ok": false, "error": null, "completion_tokens": 138, "reasoning_chars": 402, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-503", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-639", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-111", "ok": true, "error": null, "completion_tokens": 639, "reasoning_chars": 2144, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-289", "ok": true, "error": null, "completion_tokens": 435, "reasoning_chars": 1310, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-705", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-373", "ok": true, "error": null, "completion_tokens": 533, "reasoning_chars": 1562, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-363", "ok": true, "error": null, "completion_tokens": 583, "reasoning_chars": 1743, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-792", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-113", "ok": false, "error": null, "completion_tokens": 199, "reasoning_chars": 591, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-74", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 662, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-634", "ok": true, "error": null, "completion_tokens": 508, "reasoning_chars": 1395, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-804", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 1428, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-626", "ok": true, "error": null, "completion_tokens": 310, "reasoning_chars": 957, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-676", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-174", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-455", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 598, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-725", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-296", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-535", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-172", "ok": true, "error": null, "completion_tokens": 335, "reasoning_chars": 956, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-23", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-43", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-38", "ok": true, "error": null, "completion_tokens": 606, "reasoning_chars": 1766, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-97", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-657", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-814", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-346", "ok": true, "error": null, "completion_tokens": 252, "reasoning_chars": 834, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-365", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-384", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-677", "ok": true, "error": null, "completion_tokens": 505, "reasoning_chars": 1521, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-486", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-55", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-680", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 673, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-510", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-128", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 807, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-805", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-394", "ok": true, "error": null, "completion_tokens": 472, "reasoning_chars": 1348, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-435", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 890, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-285", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-687", "ok": true, "error": null, "completion_tokens": 417, "reasoning_chars": 1210, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-588", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-100", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-399", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-266", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-693", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-532", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-558", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 731, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-319", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-459", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 1035, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-675", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-129", "ok": true, "error": null, "completion_tokens": 821, "reasoning_chars": 3320, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-331", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-12", "ok": false, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-164", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-306", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 525, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-275", "ok": false, "error": null, "completion_tokens": 240, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-418", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-264", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-94", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-389", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-427", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-269", "ok": false, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-643", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-7", "ok": true, "error": null, "completion_tokens": 442, "reasoning_chars": 1166, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-419", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-616", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-699", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-473", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-523", "ok": true, "error": null, "completion_tokens": 571, "reasoning_chars": 1946, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-214", "ok": false, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-376", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 675, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-123", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-200", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-493", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-474", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 655, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-387", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-746", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-790", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-181", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-413", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-349", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-434", "ok": true, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-601", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 940, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-489", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-286", "ok": false, "error": null, "completion_tokens": 286, "reasoning_chars": 799, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-488", "ok": true, "error": null, "completion_tokens": 460, "reasoning_chars": 1322, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-759", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 769, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-273", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 755, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-59", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-315", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-425", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-58", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-691", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-497", "ok": false, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-515", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-398", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-670", "ok": true, "error": null, "completion_tokens": 980, "reasoning_chars": 1904, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-170", "ok": false, "error": null, "completion_tokens": 463, "reasoning_chars": 1174, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-334", "ok": true, "error": null, "completion_tokens": 643, "reasoning_chars": 1895, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-307", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-495", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 1166, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-597", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-775", "ok": true, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-186", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-733", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-308", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-5", "ok": false, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-231", "ok": true, "error": null, "completion_tokens": 273, "reasoning_chars": 763, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-356", "ok": false, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-810", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-270", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-13", "ok": false, "error": null, "completion_tokens": 58, "reasoning_chars": 238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-114", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 914, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-4", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 1382, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-417", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 579, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-750", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "gsm8k": { "task": "gsm8k", "n": 100, "hits": 95, "accuracy_pct": 95.0, "accuracy_ex_trunc_pct": 95.0, "truncated": 0, "completion_tokens_p50": 192, "completion_tokens_p90": 624, "completion_tokens_max": 5481, "reasoning_chars_p50": 365, "reasoning_chars_p90": 1717, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1417.1, "gen_tokens": 35938, "tok_per_s": 25.4, "details": [ { "id": "gsm8k-144", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 900, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-627", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 565, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-195", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 339, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-201", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-730", "ok": true, "error": null, "completion_tokens": 131, "reasoning_chars": 251, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-166", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 364, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1141", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 270, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-832", "ok": true, "error": null, "completion_tokens": 679, "reasoning_chars": 1962, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-53", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1042", "ok": false, "error": null, "completion_tokens": 988, "reasoning_chars": 2939, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-609", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-318", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-892", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 350, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-204", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 235, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1120", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 508, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1040", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 637, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-556", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1015", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 365, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-206", "ok": true, "error": null, "completion_tokens": 844, "reasoning_chars": 2713, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1058", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1044", "ok": true, "error": null, "completion_tokens": 192, "reasoning_chars": 419, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-131", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 329, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1154", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-158", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 225, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1149", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 298, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1156", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 317, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-60", "ok": true, "error": null, "completion_tokens": 252, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-918", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1010", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1032", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 802, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1053", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 433, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-577", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 301, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-935", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 438, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-692", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-153", "ok": true, "error": null, "completion_tokens": 536, "reasoning_chars": 1354, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-40", "ok": true, "error": null, "completion_tokens": 700, "reasoning_chars": 2320, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-613", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-875", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 441, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-189", "ok": true, "error": null, "completion_tokens": 1161, "reasoning_chars": 3222, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-800", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 256, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-169", "ok": true, "error": null, "completion_tokens": 192, "reasoning_chars": 436, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1169", "ok": true, "error": null, "completion_tokens": 624, "reasoning_chars": 1705, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-211", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 335, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1183", "ok": true, "error": null, "completion_tokens": 4513, "reasoning_chars": 17506, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-193", "ok": true, "error": null, "completion_tokens": 338, "reasoning_chars": 849, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-230", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 456, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-787", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 384, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-83", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 170, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1318", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 261, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-733", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 221, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1205", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 335, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-88", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 224, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-509", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 1036, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-311", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 292, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-874", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-287", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 482, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1055", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1246", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 210, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-975", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 564, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-280", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 283, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-541", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 592, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-974", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 310, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-827", "ok": true, "error": null, "completion_tokens": 686, "reasoning_chars": 1717, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-125", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 1753, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-453", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-942", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 503, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1199", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-884", "ok": true, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-855", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 553, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1270", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 314, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-349", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 287, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1231", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 572, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1106", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 361, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-803", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 548, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-922", "ok": false, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-152", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 640, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-920", "ok": true, "error": null, "completion_tokens": 218, "reasoning_chars": 386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-207", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-297", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-786", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 312, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-101", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 423, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1301", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 458, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-27", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 423, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1005", "ok": true, "error": null, "completion_tokens": 1317, "reasoning_chars": 3190, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-659", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 432, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-340", "ok": true, "error": null, "completion_tokens": 5481, "reasoning_chars": 20299, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-288", "ok": false, "error": null, "completion_tokens": 2728, "reasoning_chars": 9421, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-763", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-840", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 331, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-742", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 283, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-222", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-468", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 765, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-220", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 372, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-662", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 285, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-269", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 326, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1193", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 654, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-270", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 222, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-114", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 874, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-728", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 465, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-396", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 321, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "math500": { "task": "math500", "n": 100, "hits": 75, "accuracy_pct": 75.0, "accuracy_ex_trunc_pct": 75.0, "truncated": 0, "completion_tokens_p50": 388, "completion_tokens_p90": 3754, "completion_tokens_max": 16866, "reasoning_chars_p50": 723, "reasoning_chars_p90": 7721, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1518.5, "gen_tokens": 129757, "tok_per_s": 85.5, "details": [ { "id": "math-test/algebra/1553.json", "ok": true, "error": null, "completion_tokens": 449, "reasoning_chars": 663, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/538.json", "ok": true, "error": null, "completion_tokens": 716, "reasoning_chars": 923, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/691.json", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 597, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1090.json", "ok": true, "error": null, "completion_tokens": 930, "reasoning_chars": 1932, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/116.json", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 235, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1255.json", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 1150, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1143.json", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 978, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1201.json", "ok": true, "error": null, "completion_tokens": 3683, "reasoning_chars": 7538, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/964.json", "ok": false, "error": null, "completion_tokens": 2096, "reasoning_chars": 3997, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/662.json", "ok": false, "error": null, "completion_tokens": 3308, "reasoning_chars": 5337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/153.json", "ok": false, "error": null, "completion_tokens": 268, "reasoning_chars": 332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/276.json", "ok": true, "error": null, "completion_tokens": 459, "reasoning_chars": 626, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1840.json", "ok": true, "error": null, "completion_tokens": 374, "reasoning_chars": 869, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1072.json", "ok": false, "error": null, "completion_tokens": 623, "reasoning_chars": 916, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1907.json", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 122, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/567.json", "ok": true, "error": null, "completion_tokens": 368, "reasoning_chars": 462, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1922.json", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1779.json", "ok": true, "error": null, "completion_tokens": 5901, "reasoning_chars": 14039, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/951.json", "ok": true, "error": null, "completion_tokens": 288, "reasoning_chars": 409, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1197.json", "ok": false, "error": null, "completion_tokens": 7367, "reasoning_chars": 13725, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/853.json", "ok": true, "error": null, "completion_tokens": 1608, "reasoning_chars": 3153, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/516.json", "ok": true, "error": null, "completion_tokens": 1026, "reasoning_chars": 1873, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1114.json", "ok": false, "error": null, "completion_tokens": 302, "reasoning_chars": 852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1065.json", "ok": true, "error": null, "completion_tokens": 1020, "reasoning_chars": 1420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1232.json", "ok": false, "error": null, "completion_tokens": 357, "reasoning_chars": 339, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/769.json", "ok": true, "error": null, "completion_tokens": 587, "reasoning_chars": 610, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1791.json", "ok": false, "error": null, "completion_tokens": 3292, "reasoning_chars": 7721, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/246.json", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 383, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1166.json", "ok": true, "error": null, "completion_tokens": 3754, "reasoning_chars": 6836, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1410.json", "ok": false, "error": null, "completion_tokens": 432, "reasoning_chars": 729, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1686.json", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 406, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/754.json", "ok": false, "error": null, "completion_tokens": 734, "reasoning_chars": 1427, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/477.json", "ok": true, "error": null, "completion_tokens": 494, "reasoning_chars": 1731, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2680.json", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 265, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/23957.json", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 245, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1865.json", "ok": true, "error": null, "completion_tokens": 580, "reasoning_chars": 974, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1014.json", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 725, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/431.json", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 384, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1425.json", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2584.json", "ok": false, "error": null, "completion_tokens": 243, "reasoning_chars": 463, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2470.json", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 296, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/547.json", "ok": true, "error": null, "completion_tokens": 3759, "reasoning_chars": 10937, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1105.json", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/347.json", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 355, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1257.json", "ok": true, "error": null, "completion_tokens": 597, "reasoning_chars": 1647, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1126.json", "ok": true, "error": null, "completion_tokens": 545, "reasoning_chars": 818, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/239.json", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 288, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1961.json", "ok": false, "error": null, "completion_tokens": 387, "reasoning_chars": 943, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1388.json", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 271, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1078.json", "ok": true, "error": null, "completion_tokens": 620, "reasoning_chars": 1068, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2157.json", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 657, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/339.json", "ok": true, "error": null, "completion_tokens": 1756, "reasoning_chars": 6709, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1761.json", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 384, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1960.json", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 267, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2086.json", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 492, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1462.json", "ok": true, "error": null, "completion_tokens": 4880, "reasoning_chars": 8340, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1060.json", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 424, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1055.json", "ok": true, "error": null, "completion_tokens": 2591, "reasoning_chars": 7341, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2517.json", "ok": false, "error": null, "completion_tokens": 663, "reasoning_chars": 1218, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/996.json", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 356, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/893.json", "ok": true, "error": null, "completion_tokens": 577, "reasoning_chars": 685, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/51.json", "ok": false, "error": null, "completion_tokens": 479, "reasoning_chars": 1353, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1291.json", "ok": true, "error": null, "completion_tokens": 405, "reasoning_chars": 574, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/694.json", "ok": true, "error": null, "completion_tokens": 414, "reasoning_chars": 680, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/711.json", "ok": false, "error": null, "completion_tokens": 16866, "reasoning_chars": 55768, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1655.json", "ok": false, "error": null, "completion_tokens": 306, "reasoning_chars": 377, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1936.json", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 245, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1572.json", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 775, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/509.json", "ok": false, "error": null, "completion_tokens": 526, "reasoning_chars": 723, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/864.json", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 260, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1405.json", "ok": true, "error": null, "completion_tokens": 1372, "reasoning_chars": 2635, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/192.json", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 191, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2102.json", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 443, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/843.json", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 847, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/187.json", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 699, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/612.json", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 1518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/615.json", "ok": true, "error": null, "completion_tokens": 5828, "reasoning_chars": 16684, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1908.json", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 390, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/928.json", "ok": true, "error": null, "completion_tokens": 685, "reasoning_chars": 1344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/505.json", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 209, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1930.json", "ok": false, "error": null, "completion_tokens": 5717, "reasoning_chars": 11976, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1934.json", "ok": true, "error": null, "completion_tokens": 349, "reasoning_chars": 350, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1139.json", "ok": true, "error": null, "completion_tokens": 11238, "reasoning_chars": 33348, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1000.json", "ok": true, "error": null, "completion_tokens": 629, "reasoning_chars": 1398, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/183.json", "ok": true, "error": null, "completion_tokens": 812, "reasoning_chars": 1588, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1275.json", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 665, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2700.json", "ok": false, "error": null, "completion_tokens": 1688, "reasoning_chars": 4774, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2257.json", "ok": false, "error": null, "completion_tokens": 342, "reasoning_chars": 483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/14.json", "ok": false, "error": null, "completion_tokens": 374, "reasoning_chars": 1175, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1300.json", "ok": false, "error": null, "completion_tokens": 841, "reasoning_chars": 1301, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/515.json", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1423.json", "ok": false, "error": null, "completion_tokens": 214, "reasoning_chars": 395, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1102.json", "ok": false, "error": null, "completion_tokens": 727, "reasoning_chars": 1695, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1646.json", "ok": true, "error": null, "completion_tokens": 3937, "reasoning_chars": 11023, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1272.json", "ok": false, "error": null, "completion_tokens": 132, "reasoning_chars": 181, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/25149.json", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/697.json", "ok": true, "error": null, "completion_tokens": 8777, "reasoning_chars": 17297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2036.json", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2037.json", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/353.json", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 542, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "bbh": { "task": "bbh", "n": 150, "hits": 113, "accuracy_pct": 75.33, "accuracy_ex_trunc_pct": 75.33, "truncated": 0, "completion_tokens_p50": 188, "completion_tokens_p90": 648, "completion_tokens_max": 7647, "reasoning_chars_p50": 573, "reasoning_chars_p90": 1737, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1374.3, "gen_tokens": 56093, "tok_per_s": 40.8, "details": [ { "id": "bbh-sports_understanding-101", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 740, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-60", "ok": true, "error": null, "completion_tokens": 383, "reasoning_chars": 1128, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-113", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-31", "ok": false, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-144", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-28", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 1030, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-116", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 294, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-67", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 738, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-3", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 736, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-29", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-192", "ok": true, "error": null, "completion_tokens": 875, "reasoning_chars": 1951, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-196", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 773, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-10", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 696, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-83", "ok": true, "error": null, "completion_tokens": 64, "reasoning_chars": 267, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-76", "ok": true, "error": null, "completion_tokens": 1908, "reasoning_chars": 7764, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-212", "ok": true, "error": null, "completion_tokens": 457, "reasoning_chars": 1354, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-87", "ok": true, "error": null, "completion_tokens": 474, "reasoning_chars": 1236, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-1", "ok": true, "error": null, "completion_tokens": 519, "reasoning_chars": 1001, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-140", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 121, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-8", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-138", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 1163, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-221", "ok": true, "error": null, "completion_tokens": 672, "reasoning_chars": 1737, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-136", "ok": true, "error": null, "completion_tokens": 623, "reasoning_chars": 1406, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-14", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 62, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-0", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 146, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-207", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-32", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 1092, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-83", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-220", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-234", "ok": true, "error": null, "completion_tokens": 648, "reasoning_chars": 1915, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-61", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-21", "ok": true, "error": null, "completion_tokens": 275, "reasoning_chars": 662, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-31", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 1214, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-13", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-211", "ok": false, "error": null, "completion_tokens": 96, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-37", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 601, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-225", "ok": true, "error": null, "completion_tokens": 1089, "reasoning_chars": 2624, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-90", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-101", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-179", "ok": true, "error": null, "completion_tokens": 42, "reasoning_chars": 63, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-173", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 379, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-105", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 69, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-88", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 763, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-68", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-185", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-227", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-129", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 29, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-238", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-202", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 847, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-93", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 1076, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-93", "ok": false, "error": null, "completion_tokens": 120, "reasoning_chars": 155, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-161", "ok": false, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-46", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 421, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-227", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-158", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 22, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-205", "ok": false, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-89", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 164, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-177", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 548, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-65", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-72", "ok": true, "error": null, "completion_tokens": 268, "reasoning_chars": 495, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-49", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 323, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-54", "ok": true, "error": null, "completion_tokens": 1143, "reasoning_chars": 3083, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-64", "ok": false, "error": null, "completion_tokens": 538, "reasoning_chars": 2104, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-86", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-108", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 194, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-194", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 1130, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-194", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 412, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-75", "ok": false, "error": null, "completion_tokens": 7647, "reasoning_chars": 27041, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-106", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 632, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-81", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 771, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-242", "ok": true, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-150", "ok": true, "error": null, "completion_tokens": 455, "reasoning_chars": 1703, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-25", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 1002, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-139", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 605, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-12", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 935, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-178", "ok": true, "error": null, "completion_tokens": 506, "reasoning_chars": 786, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-160", "ok": true, "error": null, "completion_tokens": 499, "reasoning_chars": 2032, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-115", "ok": true, "error": null, "completion_tokens": 4745, "reasoning_chars": 17759, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-44", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 1176, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-189", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 677, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-102", "ok": true, "error": null, "completion_tokens": 508, "reasoning_chars": 758, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-38", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-141", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 1530, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-38", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 1104, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-168", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-212", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-245", "ok": true, "error": null, "completion_tokens": 228, "reasoning_chars": 946, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-222", "ok": true, "error": null, "completion_tokens": 536, "reasoning_chars": 1027, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-244", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 661, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-35", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 550, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-126", "ok": false, "error": null, "completion_tokens": 859, "reasoning_chars": 2721, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-90", "ok": true, "error": null, "completion_tokens": 1099, "reasoning_chars": 3987, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-5", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 597, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-241", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-234", "ok": true, "error": null, "completion_tokens": 632, "reasoning_chars": 1501, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-233", "ok": true, "error": null, "completion_tokens": 465, "reasoning_chars": 346, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-5", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-152", "ok": true, "error": null, "completion_tokens": 440, "reasoning_chars": 1312, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-9", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 125, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-209", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 143, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-171", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 949, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-160", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-166", "ok": false, "error": null, "completion_tokens": 116, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-213", "ok": true, "error": null, "completion_tokens": 1073, "reasoning_chars": 3564, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-240", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 77, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-95", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 76, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-227", "ok": true, "error": null, "completion_tokens": 531, "reasoning_chars": 1555, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-96", "ok": false, "error": null, "completion_tokens": 143, "reasoning_chars": 644, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-33", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-4", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 1327, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-180", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-74", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 1023, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-88", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 459, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-10", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 642, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-191", "ok": false, "error": null, "completion_tokens": 2955, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-71", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-150", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-188", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 194, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-191", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-203", "ok": false, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-237", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 824, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-24", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 99, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-27", "ok": true, "error": null, "completion_tokens": 351, "reasoning_chars": 1098, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-177", "ok": true, "error": null, "completion_tokens": 349, "reasoning_chars": 1020, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-192", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-242", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-51", "ok": false, "error": null, "completion_tokens": 603, "reasoning_chars": 2137, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-96", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-100", "ok": true, "error": null, "completion_tokens": 335, "reasoning_chars": 1308, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-204", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-58", "ok": true, "error": null, "completion_tokens": 455, "reasoning_chars": 1430, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-157", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 2, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-104", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-47", "ok": true, "error": null, "completion_tokens": 422, "reasoning_chars": 1172, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-150", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 108, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-23", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 163, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-68", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-174", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 538, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-108", "ok": true, "error": null, "completion_tokens": 899, "reasoning_chars": 1599, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-203", "ok": false, "error": null, "completion_tokens": 674, "reasoning_chars": 995, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-115", "ok": true, "error": null, "completion_tokens": 518, "reasoning_chars": 791, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-223", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 1062, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-221", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-5", "ok": true, "error": null, "completion_tokens": 412, "reasoning_chars": 1232, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-233", "ok": true, "error": null, "completion_tokens": 1530, "reasoning_chars": 1580, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-3", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-119", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 573, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-197", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-103", "ok": true, "error": null, "completion_tokens": 2027, "reasoning_chars": 8717, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-178", "ok": false, "error": null, "completion_tokens": 445, "reasoning_chars": 1947, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "humaneval": { "task": "humaneval", "n": 164, "hits": 147, "accuracy_pct": 89.63, "accuracy_ex_trunc_pct": 92.45, "truncated": 5, "completion_tokens_p50": 583, "completion_tokens_p90": 3385, "completion_tokens_max": 32768, "reasoning_chars_p50": 1500, "reasoning_chars_p90": 9119, "reasoning_leaked": null, "empty": 5, "length_limited": 5, "errors": 0, "elapsed_s": 2231.9, "gen_tokens": 386060, "tok_per_s": 173.0, "details": [ { "id": "HumanEval/0", "ok": false, "error": null, "completion_tokens": 433, "reasoning_chars": 1664, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/1", "ok": true, "error": null, "completion_tokens": 1484, "reasoning_chars": 5995, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/2", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 1136, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/3", "ok": true, "error": null, "completion_tokens": 282, "reasoning_chars": 1080, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/4", "ok": false, "error": null, "completion_tokens": 608, "reasoning_chars": 2455, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/5", "ok": true, "error": null, "completion_tokens": 347, "reasoning_chars": 879, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/6", "ok": true, "error": null, "completion_tokens": 907, "reasoning_chars": 3218, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/7", "ok": false, "error": null, "completion_tokens": 221, "reasoning_chars": 606, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/8", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 784, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/9", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 1249, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/10", "ok": true, "error": null, "completion_tokens": 2185, "reasoning_chars": 7698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/11", "ok": true, "error": null, "completion_tokens": 1308, "reasoning_chars": 4734, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/12", "ok": false, "error": null, "completion_tokens": 1349, "reasoning_chars": 6302, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/13", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 680, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/14", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 1644, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/15", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 710, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/16", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 484, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/17", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 806, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/18", "ok": true, "error": null, "completion_tokens": 542, "reasoning_chars": 1541, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/19", "ok": true, "error": null, "completion_tokens": 469, "reasoning_chars": 1402, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/20", "ok": false, "error": null, "completion_tokens": 703, "reasoning_chars": 2806, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/21", "ok": false, "error": null, "completion_tokens": 1166, "reasoning_chars": 4922, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/22", "ok": false, "error": null, "completion_tokens": 4587, "reasoning_chars": 17468, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/23", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 614, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/24", "ok": true, "error": null, "completion_tokens": 1760, "reasoning_chars": 5825, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/25", "ok": true, "error": null, "completion_tokens": 1471, "reasoning_chars": 5023, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/26", "ok": false, "error": null, "completion_tokens": 396, "reasoning_chars": 1500, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/27", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/28", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 417, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/29", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 573, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/30", "ok": true, "error": null, "completion_tokens": 383, "reasoning_chars": 778, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/31", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 521, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/32", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 109184, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/33", "ok": true, "error": null, "completion_tokens": 637, "reasoning_chars": 1412, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/34", "ok": true, "error": null, "completion_tokens": 332, "reasoning_chars": 934, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/35", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 613, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/36", "ok": true, "error": null, "completion_tokens": 581, "reasoning_chars": 1127, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/37", "ok": true, "error": null, "completion_tokens": 583, "reasoning_chars": 1457, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/38", "ok": false, "error": null, "completion_tokens": 359, "reasoning_chars": 978, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/39", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 59311, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/40", "ok": true, "error": null, "completion_tokens": 830, "reasoning_chars": 2077, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/41", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 557, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/42", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/43", "ok": true, "error": null, "completion_tokens": 394, "reasoning_chars": 854, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/44", "ok": true, "error": null, "completion_tokens": 1536, "reasoning_chars": 4863, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/45", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 197, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/46", "ok": true, "error": null, "completion_tokens": 1442, "reasoning_chars": 3296, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/47", "ok": false, "error": null, "completion_tokens": 16243, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/48", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/49", "ok": true, "error": null, "completion_tokens": 405, "reasoning_chars": 984, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/50", "ok": false, "error": null, "completion_tokens": 238, "reasoning_chars": 742, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/51", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 811, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/52", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 520, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/53", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 444, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/54", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 1006, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/55", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 909, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/56", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 825, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/57", "ok": true, "error": null, "completion_tokens": 642, "reasoning_chars": 1795, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/58", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/59", "ok": true, "error": null, "completion_tokens": 633, "reasoning_chars": 1300, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/60", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 711, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/61", "ok": true, "error": null, "completion_tokens": 254, "reasoning_chars": 488, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/62", "ok": true, "error": null, "completion_tokens": 310, "reasoning_chars": 702, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/63", "ok": true, "error": null, "completion_tokens": 606, "reasoning_chars": 1056, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/64", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 969, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/65", "ok": true, "error": null, "completion_tokens": 848, "reasoning_chars": 2745, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/66", "ok": true, "error": null, "completion_tokens": 496, "reasoning_chars": 1147, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/67", "ok": true, "error": null, "completion_tokens": 464, "reasoning_chars": 825, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/68", "ok": true, "error": null, "completion_tokens": 791, "reasoning_chars": 2920, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/69", "ok": true, "error": null, "completion_tokens": 261, "reasoning_chars": 900, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/70", "ok": true, "error": null, "completion_tokens": 626, "reasoning_chars": 1690, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/71", "ok": true, "error": null, "completion_tokens": 2760, "reasoning_chars": 8275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/72", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 562, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/73", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 1028, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/74", "ok": true, "error": null, "completion_tokens": 372, "reasoning_chars": 1089, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/75", "ok": true, "error": null, "completion_tokens": 3385, "reasoning_chars": 9938, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/76", "ok": true, "error": null, "completion_tokens": 6444, "reasoning_chars": 17595, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/77", "ok": true, "error": null, "completion_tokens": 1584, "reasoning_chars": 4172, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/78", "ok": true, "error": null, "completion_tokens": 320, "reasoning_chars": 1082, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/79", "ok": true, "error": null, "completion_tokens": 360, "reasoning_chars": 758, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/80", "ok": true, "error": null, "completion_tokens": 421, "reasoning_chars": 818, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/81", "ok": true, "error": null, "completion_tokens": 2729, "reasoning_chars": 6414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/82", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 662, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/83", "ok": true, "error": null, "completion_tokens": 851, "reasoning_chars": 2072, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/84", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/85", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 691, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/86", "ok": true, "error": null, "completion_tokens": 699, "reasoning_chars": 2068, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/87", "ok": true, "error": null, "completion_tokens": 842, "reasoning_chars": 1853, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/88", "ok": true, "error": null, "completion_tokens": 723, "reasoning_chars": 1845, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/89", "ok": true, "error": null, "completion_tokens": 557, "reasoning_chars": 1145, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/90", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 873, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/91", "ok": true, "error": null, "completion_tokens": 2085, "reasoning_chars": 7358, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/92", "ok": true, "error": null, "completion_tokens": 2090, "reasoning_chars": 7632, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/93", "ok": true, "error": null, "completion_tokens": 2077, "reasoning_chars": 6746, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/94", "ok": true, "error": null, "completion_tokens": 1194, "reasoning_chars": 2044, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/95", "ok": true, "error": null, "completion_tokens": 1405, "reasoning_chars": 4604, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/96", "ok": true, "error": null, "completion_tokens": 19790, "reasoning_chars": 42310, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/97", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 482, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/98", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 744, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/99", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 56038, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/100", "ok": true, "error": null, "completion_tokens": 417, "reasoning_chars": 928, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/101", "ok": true, "error": null, "completion_tokens": 349, "reasoning_chars": 901, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/102", "ok": true, "error": null, "completion_tokens": 3321, "reasoning_chars": 8301, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/103", "ok": true, "error": null, "completion_tokens": 1636, "reasoning_chars": 4656, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/104", "ok": true, "error": null, "completion_tokens": 510, "reasoning_chars": 1509, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/105", "ok": true, "error": null, "completion_tokens": 1433, "reasoning_chars": 3513, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/106", "ok": true, "error": null, "completion_tokens": 1691, "reasoning_chars": 5161, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/107", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 715, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/108", "ok": true, "error": null, "completion_tokens": 948, "reasoning_chars": 2382, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/109", "ok": true, "error": null, "completion_tokens": 1273, "reasoning_chars": 2843, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/110", "ok": true, "error": null, "completion_tokens": 520, "reasoning_chars": 994, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/111", "ok": true, "error": null, "completion_tokens": 508, "reasoning_chars": 1140, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/112", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 559, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/113", "ok": true, "error": null, "completion_tokens": 1018, "reasoning_chars": 2620, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/114", "ok": true, "error": null, "completion_tokens": 1083, "reasoning_chars": 3126, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/115", "ok": true, "error": null, "completion_tokens": 742, "reasoning_chars": 1630, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/116", "ok": true, "error": null, "completion_tokens": 6806, "reasoning_chars": 27334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/117", "ok": true, "error": null, "completion_tokens": 767, "reasoning_chars": 1936, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/118", "ok": true, "error": null, "completion_tokens": 1213, "reasoning_chars": 3876, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/119", "ok": true, "error": null, "completion_tokens": 783, "reasoning_chars": 2022, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/120", "ok": true, "error": null, "completion_tokens": 583, "reasoning_chars": 1296, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/121", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 726, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/122", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 878, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/123", "ok": true, "error": null, "completion_tokens": 940, "reasoning_chars": 2196, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/124", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 81483, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/125", "ok": true, "error": null, "completion_tokens": 741, "reasoning_chars": 2155, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/126", "ok": true, "error": null, "completion_tokens": 935, "reasoning_chars": 2249, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/127", "ok": true, "error": null, "completion_tokens": 775, "reasoning_chars": 2381, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/128", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 1090, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/129", "ok": true, "error": null, "completion_tokens": 11223, "reasoning_chars": 41060, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/130", "ok": true, "error": null, "completion_tokens": 8813, "reasoning_chars": 21429, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/131", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 760, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/132", "ok": true, "error": null, "completion_tokens": 5466, "reasoning_chars": 20905, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/133", "ok": true, "error": null, "completion_tokens": 477, "reasoning_chars": 1220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/134", "ok": true, "error": null, "completion_tokens": 1607, "reasoning_chars": 5399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/135", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 945, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/136", "ok": true, "error": null, "completion_tokens": 528, "reasoning_chars": 1317, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/137", "ok": true, "error": null, "completion_tokens": 763, "reasoning_chars": 2070, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/138", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 610, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/139", "ok": true, "error": null, "completion_tokens": 569, "reasoning_chars": 1611, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/140", "ok": true, "error": null, "completion_tokens": 1472, "reasoning_chars": 4760, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/141", "ok": true, "error": null, "completion_tokens": 1508, "reasoning_chars": 4209, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/142", "ok": true, "error": null, "completion_tokens": 1200, "reasoning_chars": 2726, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/143", "ok": true, "error": null, "completion_tokens": 601, "reasoning_chars": 1491, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/144", "ok": true, "error": null, "completion_tokens": 629, "reasoning_chars": 1605, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/145", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 41839, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/146", "ok": true, "error": null, "completion_tokens": 1705, "reasoning_chars": 4238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/147", "ok": true, "error": null, "completion_tokens": 4524, "reasoning_chars": 11416, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/148", "ok": true, "error": null, "completion_tokens": 671, "reasoning_chars": 1335, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/149", "ok": true, "error": null, "completion_tokens": 497, "reasoning_chars": 1143, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/150", "ok": true, "error": null, "completion_tokens": 430, "reasoning_chars": 915, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/151", "ok": true, "error": null, "completion_tokens": 828, "reasoning_chars": 2544, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/152", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/153", "ok": true, "error": null, "completion_tokens": 1258, "reasoning_chars": 3621, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/154", "ok": true, "error": null, "completion_tokens": 921, "reasoning_chars": 2497, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/155", "ok": true, "error": null, "completion_tokens": 567, "reasoning_chars": 1602, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/156", "ok": true, "error": null, "completion_tokens": 805, "reasoning_chars": 1680, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/157", "ok": true, "error": null, "completion_tokens": 19805, "reasoning_chars": 64364, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/158", "ok": false, "error": null, "completion_tokens": 612, "reasoning_chars": 1851, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/159", "ok": true, "error": null, "completion_tokens": 491, "reasoning_chars": 691, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/160", "ok": true, "error": null, "completion_tokens": 6743, "reasoning_chars": 26139, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/161", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 644, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/162", "ok": true, "error": null, "completion_tokens": 536, "reasoning_chars": 1860, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/163", "ok": true, "error": null, "completion_tokens": 3144, "reasoning_chars": 9119, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ifeval": { "task": "ifeval", "n": 100, "hits": 76, "accuracy_pct": 76.0, "accuracy_ex_trunc_pct": 76.0, "truncated": 0, "completion_tokens_p50": 570, "completion_tokens_p90": 2094, "completion_tokens_max": 8288, "reasoning_chars_p50": 1266, "reasoning_chars_p90": 5954, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1395.4, "gen_tokens": 98676, "tok_per_s": 70.7, "instruction_level_pct": 84.67, "instructions_checked": 150, "details": [ { "id": "ifeval-153", "ok": true, "error": null, "completion_tokens": 1700, "reasoning_chars": 4178, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-258", "ok": true, "error": null, "completion_tokens": 212, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-368", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-85", "ok": true, "error": null, "completion_tokens": 1863, "reasoning_chars": 5954, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-246", "ok": true, "error": null, "completion_tokens": 1544, "reasoning_chars": 3592, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-23", "ok": true, "error": null, "completion_tokens": 1565, "reasoning_chars": 4578, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-372", "ok": true, "error": null, "completion_tokens": 1218, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-387", "ok": true, "error": null, "completion_tokens": 403, "reasoning_chars": 1174, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-138", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 206, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-56", "ok": true, "error": null, "completion_tokens": 660, "reasoning_chars": 2422, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-277", "ok": true, "error": null, "completion_tokens": 680, "reasoning_chars": 1935, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-407", "ok": false, "error": null, "completion_tokens": 248, "reasoning_chars": 201, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-171", "ok": true, "error": null, "completion_tokens": 412, "reasoning_chars": 828, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-96", "ok": true, "error": null, "completion_tokens": 1632, "reasoning_chars": 1125, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-134", "ok": true, "error": null, "completion_tokens": 3361, "reasoning_chars": 8432, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-218", "ok": true, "error": null, "completion_tokens": 770, "reasoning_chars": 1266, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-330", "ok": false, "error": null, "completion_tokens": 1835, "reasoning_chars": 7191, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-6", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 1284, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-143", "ok": true, "error": null, "completion_tokens": 236, "reasoning_chars": 356, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-121", "ok": true, "error": null, "completion_tokens": 436, "reasoning_chars": 1461, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-219", "ok": true, "error": null, "completion_tokens": 456, "reasoning_chars": 1597, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-524", "ok": true, "error": null, "completion_tokens": 312, "reasoning_chars": 345, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-206", "ok": true, "error": null, "completion_tokens": 435, "reasoning_chars": 1578, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-144", "ok": true, "error": null, "completion_tokens": 705, "reasoning_chars": 1698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-447", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 982, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-492", "ok": true, "error": null, "completion_tokens": 2435, "reasoning_chars": 9167, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-485", "ok": true, "error": null, "completion_tokens": 1962, "reasoning_chars": 4193, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-220", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-412", "ok": true, "error": null, "completion_tokens": 762, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-342", "ok": false, "error": null, "completion_tokens": 401, "reasoning_chars": 422, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-222", "ok": true, "error": null, "completion_tokens": 8288, "reasoning_chars": 25793, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-131", "ok": false, "error": null, "completion_tokens": 526, "reasoning_chars": 1745, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-210", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 636, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-137", "ok": true, "error": null, "completion_tokens": 503, "reasoning_chars": 899, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-464", "ok": true, "error": null, "completion_tokens": 493, "reasoning_chars": 1801, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-503", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-446", "ok": true, "error": null, "completion_tokens": 823, "reasoning_chars": 1979, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-2", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-525", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 856, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-27", "ok": false, "error": null, "completion_tokens": 355, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-136", "ok": true, "error": null, "completion_tokens": 954, "reasoning_chars": 2251, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-286", "ok": true, "error": null, "completion_tokens": 637, "reasoning_chars": 2352, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-449", "ok": true, "error": null, "completion_tokens": 863, "reasoning_chars": 3236, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-55", "ok": true, "error": null, "completion_tokens": 1168, "reasoning_chars": 4271, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-93", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 604, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-399", "ok": true, "error": null, "completion_tokens": 698, "reasoning_chars": 2318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-19", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 541, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-229", "ok": true, "error": null, "completion_tokens": 614, "reasoning_chars": 2078, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-181", "ok": false, "error": null, "completion_tokens": 1014, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-362", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-289", "ok": true, "error": null, "completion_tokens": 825, "reasoning_chars": 3169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-356", "ok": false, "error": null, "completion_tokens": 246, "reasoning_chars": 540, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-115", "ok": false, "error": null, "completion_tokens": 1078, "reasoning_chars": 4372, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-404", "ok": false, "error": null, "completion_tokens": 768, "reasoning_chars": 1560, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-243", "ok": false, "error": null, "completion_tokens": 2587, "reasoning_chars": 7798, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-90", "ok": true, "error": null, "completion_tokens": 764, "reasoning_chars": 1948, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-54", "ok": true, "error": null, "completion_tokens": 1627, "reasoning_chars": 5889, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-276", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 308, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-308", "ok": true, "error": null, "completion_tokens": 569, "reasoning_chars": 1665, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-64", "ok": false, "error": null, "completion_tokens": 2003, "reasoning_chars": 7878, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-260", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 758, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-337", "ok": true, "error": null, "completion_tokens": 373, "reasoning_chars": 1050, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-510", "ok": true, "error": null, "completion_tokens": 781, "reasoning_chars": 823, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-119", "ok": true, "error": null, "completion_tokens": 4265, "reasoning_chars": 16483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-5", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 815, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-468", "ok": true, "error": null, "completion_tokens": 3932, "reasoning_chars": 10602, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-418", "ok": true, "error": null, "completion_tokens": 505, "reasoning_chars": 1965, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-476", "ok": true, "error": null, "completion_tokens": 2887, "reasoning_chars": 5594, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-49", "ok": true, "error": null, "completion_tokens": 5151, "reasoning_chars": 13955, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-336", "ok": false, "error": null, "completion_tokens": 48, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-422", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 34, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-417", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 603, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-347", "ok": false, "error": null, "completion_tokens": 1661, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-101", "ok": true, "error": null, "completion_tokens": 677, "reasoning_chars": 518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-209", "ok": true, "error": null, "completion_tokens": 600, "reasoning_chars": 1852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-341", "ok": true, "error": null, "completion_tokens": 1590, "reasoning_chars": 4513, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-533", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 625, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-158", "ok": true, "error": null, "completion_tokens": 570, "reasoning_chars": 1907, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-321", "ok": true, "error": null, "completion_tokens": 664, "reasoning_chars": 2109, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-311", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-50", "ok": false, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-518", "ok": true, "error": null, "completion_tokens": 1309, "reasoning_chars": 1575, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-374", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 1428, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-317", "ok": false, "error": null, "completion_tokens": 368, "reasoning_chars": 1343, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-512", "ok": false, "error": null, "completion_tokens": 1096, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-102", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-72", "ok": false, "error": null, "completion_tokens": 785, "reasoning_chars": 2969, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-33", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 1, "skipped": false }, { "id": "ifeval-263", "ok": true, "error": null, "completion_tokens": 426, "reasoning_chars": 804, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-501", "ok": true, "error": null, "completion_tokens": 1646, "reasoning_chars": 4962, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-4", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 595, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-444", "ok": false, "error": null, "completion_tokens": 3264, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-228", "ok": true, "error": null, "completion_tokens": 2094, "reasoning_chars": 260, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-299", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 560, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-278", "ok": false, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-154", "ok": true, "error": null, "completion_tokens": 744, "reasoning_chars": 2886, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-301", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 1044, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-161", "ok": true, "error": null, "completion_tokens": 4248, "reasoning_chars": 8701, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-394", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 1, "skipped": false }, { "id": "ifeval-159", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 1129, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false } ] } } }, "ablated-w4a16": { "started": "2026-10-10 14:32:00", "finished": "2026-10-10 15:06:32", "seed": 20260816, "thinking": true, "tasks": { "mmlu": { "task": "mmlu", "n": 150, "hits": 114, "accuracy_pct": 76.0, "accuracy_ex_trunc_pct": 77.03, "truncated": 2, "completion_tokens_p50": 230, "completion_tokens_p90": 920, "completion_tokens_max": 32768, "reasoning_chars_p50": 652, "reasoning_chars_p90": 2348, "reasoning_leaked": null, "empty": 2, "length_limited": 2, "errors": 0, "elapsed_s": 1341.9, "gen_tokens": 128545, "tok_per_s": 95.8, "details": [ { "id": "mmlu-12325", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-154", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 180, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12844", "ok": true, "error": null, "completion_tokens": 476, "reasoning_chars": 1526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1153", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 635, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11089", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 2344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12782", "ok": true, "error": null, "completion_tokens": 571, "reasoning_chars": 2960, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5422", "ok": false, "error": null, "completion_tokens": 167, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12363", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12556", "ok": false, "error": null, "completion_tokens": 236, "reasoning_chars": 1118, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13163", "ok": true, "error": null, "completion_tokens": 747, "reasoning_chars": 2344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-270", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 980, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6943", "ok": true, "error": null, "completion_tokens": 517, "reasoning_chars": 2209, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8330", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 609, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9276", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 525, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7584", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 230, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12819", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6163", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 458, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4099", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 774, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10389", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 252, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1539", "ok": false, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12013", "ok": true, "error": null, "completion_tokens": 768, "reasoning_chars": 2502, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7814", "ok": false, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1106", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 82, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1616", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 538, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-555", "ok": true, "error": null, "completion_tokens": 468, "reasoning_chars": 1404, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3698", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 885, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-618", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 658, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9842", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 881, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4229", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1022", "ok": true, "error": null, "completion_tokens": 268, "reasoning_chars": 989, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6421", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 706, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1997", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-8941", "ok": false, "error": null, "completion_tokens": 368, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-565", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 192, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11364", "ok": true, "error": null, "completion_tokens": 965, "reasoning_chars": 2269, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5959", "ok": true, "error": null, "completion_tokens": 381, "reasoning_chars": 1157, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13066", "ok": false, "error": null, "completion_tokens": 364, "reasoning_chars": 1595, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3122", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 728, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3722", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13839", "ok": false, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12570", "ok": true, "error": null, "completion_tokens": 623, "reasoning_chars": 2571, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9959", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-10791", "ok": false, "error": null, "completion_tokens": 2073, "reasoning_chars": 8489, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12268", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9051", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 650, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12887", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2667", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 351, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3906", "ok": true, "error": null, "completion_tokens": 416, "reasoning_chars": 1390, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10304", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12927", "ok": false, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6337", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 136, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3907", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 144, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12888", "ok": false, "error": null, "completion_tokens": 165, "reasoning_chars": 870, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10629", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 534, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5801", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 322, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2936", "ok": true, "error": null, "completion_tokens": 596, "reasoning_chars": 1955, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7389", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9532", "ok": false, "error": null, "completion_tokens": 229, "reasoning_chars": 565, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8705", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 976, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11776", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 154751, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlu-6836", "ok": false, "error": null, "completion_tokens": 214, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7856", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 605, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9697", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 1693, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10825", "ok": false, "error": null, "completion_tokens": 540, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11178", "ok": true, "error": null, "completion_tokens": 1548, "reasoning_chars": 6165, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11420", "ok": true, "error": null, "completion_tokens": 934, "reasoning_chars": 2220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4658", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 1002, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5403", "ok": false, "error": null, "completion_tokens": 498, "reasoning_chars": 1757, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6160", "ok": true, "error": null, "completion_tokens": 429, "reasoning_chars": 1325, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6872", "ok": false, "error": null, "completion_tokens": 5769, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9738", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 735, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3076", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 1274, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13496", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 866, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11289", "ok": false, "error": null, "completion_tokens": 1210, "reasoning_chars": 4285, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-222", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12180", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 776, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1143", "ok": false, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13086", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 635, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7038", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13587", "ok": false, "error": null, "completion_tokens": 258, "reasoning_chars": 588, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12349", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 719, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11747", "ok": false, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9848", "ok": false, "error": null, "completion_tokens": 484, "reasoning_chars": 2223, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3438", "ok": true, "error": null, "completion_tokens": 477, "reasoning_chars": 1307, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-89", "ok": true, "error": null, "completion_tokens": 230, "reasoning_chars": 928, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4253", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5460", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2494", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 109788, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlu-11149", "ok": false, "error": null, "completion_tokens": 1014, "reasoning_chars": 3037, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-718", "ok": true, "error": null, "completion_tokens": 433, "reasoning_chars": 1219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5330", "ok": false, "error": null, "completion_tokens": 163, "reasoning_chars": 754, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10716", "ok": true, "error": null, "completion_tokens": 1893, "reasoning_chars": 6258, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5636", "ok": false, "error": null, "completion_tokens": 736, "reasoning_chars": 2036, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11832", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-95", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 1253, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13960", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 482, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2072", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 17, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12030", "ok": false, "error": null, "completion_tokens": 2162, "reasoning_chars": 8931, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2931", "ok": true, "error": null, "completion_tokens": 246, "reasoning_chars": 542, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2441", "ok": true, "error": null, "completion_tokens": 426, "reasoning_chars": 434, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13666", "ok": false, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6575", "ok": true, "error": null, "completion_tokens": 137, "reasoning_chars": 609, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11625", "ok": true, "error": null, "completion_tokens": 920, "reasoning_chars": 2853, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13283", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-4262", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 591, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12931", "ok": true, "error": null, "completion_tokens": 282, "reasoning_chars": 993, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12150", "ok": true, "error": null, "completion_tokens": 757, "reasoning_chars": 2296, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3678", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4593", "ok": false, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1563", "ok": true, "error": null, "completion_tokens": 526, "reasoning_chars": 1623, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13396", "ok": true, "error": null, "completion_tokens": 490, "reasoning_chars": 1390, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8049", "ok": true, "error": null, "completion_tokens": 131, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-8396", "ok": true, "error": null, "completion_tokens": 437, "reasoning_chars": 1245, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8493", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 1017, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13514", "ok": true, "error": null, "completion_tokens": 408, "reasoning_chars": 1257, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3000", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 152, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1635", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1391", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 571, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-627", "ok": false, "error": null, "completion_tokens": 5847, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1573", "ok": true, "error": null, "completion_tokens": 40, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6731", "ok": false, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11665", "ok": true, "error": null, "completion_tokens": 1927, "reasoning_chars": 9394, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2914", "ok": true, "error": null, "completion_tokens": 490, "reasoning_chars": 1741, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6875", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2184", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12801", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6717", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12464", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 666, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2787", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 935, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11549", "ok": true, "error": null, "completion_tokens": 943, "reasoning_chars": 3454, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-842", "ok": true, "error": null, "completion_tokens": 744, "reasoning_chars": 2348, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13868", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9647", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9502", "ok": true, "error": null, "completion_tokens": 617, "reasoning_chars": 1872, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12441", "ok": true, "error": null, "completion_tokens": 429, "reasoning_chars": 1316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4392", "ok": false, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5036", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 1395, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12606", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 1577, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1400", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13979", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 617, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13634", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11226", "ok": true, "error": null, "completion_tokens": 1712, "reasoning_chars": 5962, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-625", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 1421, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-257", "ok": true, "error": null, "completion_tokens": 569, "reasoning_chars": 1267, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12853", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 652, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-451", "ok": true, "error": null, "completion_tokens": 527, "reasoning_chars": 1580, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2074", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 80, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9117", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 1304, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10748", "ok": true, "error": null, "completion_tokens": 463, "reasoning_chars": 1540, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "cmmlu": { "task": "cmmlu", "n": 150, "hits": 113, "accuracy_pct": 75.33, "accuracy_ex_trunc_pct": 75.33, "truncated": 0, "completion_tokens_p50": 156, "completion_tokens_p90": 611, "completion_tokens_max": 3012, "reasoning_chars_p50": 318, "reasoning_chars_p90": 1969, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1086.4, "gen_tokens": 37435, "tok_per_s": 34.5, "details": [ { "id": "cmmlu-chinese_teacher_qualification-63", "ok": true, "error": null, "completion_tokens": 381, "reasoning_chars": 1199, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-54", "ok": true, "error": null, "completion_tokens": 695, "reasoning_chars": 1767, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-91", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-260", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-32", "ok": true, "error": null, "completion_tokens": 53, "reasoning_chars": 54, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-29", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 1363, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-200", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-98", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-44", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 593, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-118", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-116", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-43", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 1008, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-145", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-247", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 235, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-224", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-26", "ok": true, "error": null, "completion_tokens": 611, "reasoning_chars": 1232, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-105", "ok": false, "error": null, "completion_tokens": 1255, "reasoning_chars": 1606, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-15", "ok": true, "error": null, "completion_tokens": 574, "reasoning_chars": 1518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-62", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-nutrition-74", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 1183, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-181", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 570, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-127", "ok": true, "error": null, "completion_tokens": 612, "reasoning_chars": 2420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_geography-107", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-18", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 572, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-82", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 449, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_food_culture-38", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 69, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-198", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 115, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-96", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 1031, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-73", "ok": true, "error": null, "completion_tokens": 989, "reasoning_chars": 3668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-20", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-95", "ok": true, "error": null, "completion_tokens": 468, "reasoning_chars": 778, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-88", "ok": true, "error": null, "completion_tokens": 1044, "reasoning_chars": 4315, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-192", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 361, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-66", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 801, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-100", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 416, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-136", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 253, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-16", "ok": false, "error": null, "completion_tokens": 311, "reasoning_chars": 1316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-90", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-41", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-345", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-103", "ok": false, "error": null, "completion_tokens": 184, "reasoning_chars": 296, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-93", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-88", "ok": true, "error": null, "completion_tokens": 218, "reasoning_chars": 379, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-206", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 1010, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-105", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-90", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 1116, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-190", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-international_law-39", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 914, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-155", "ok": true, "error": null, "completion_tokens": 936, "reasoning_chars": 2607, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-24", "ok": true, "error": null, "completion_tokens": 301, "reasoning_chars": 1320, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-6", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-31", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-243", "ok": true, "error": null, "completion_tokens": 688, "reasoning_chars": 1980, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-63", "ok": true, "error": null, "completion_tokens": 603, "reasoning_chars": 1980, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-135", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 1049, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-89", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 164, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-91", "ok": false, "error": null, "completion_tokens": 1192, "reasoning_chars": 4333, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-120", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 815, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-62", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-106", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 792, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-food_science-106", "ok": false, "error": null, "completion_tokens": 207, "reasoning_chars": 318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-24", "ok": true, "error": null, "completion_tokens": 1204, "reasoning_chars": 4212, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-157", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-41", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 1524, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-115", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-journalism-29", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 823, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-67", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-161", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 125, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-120", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-education-117", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-169", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 930, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-129", "ok": false, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-19", "ok": true, "error": null, "completion_tokens": 943, "reasoning_chars": 3275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-8", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 711, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-72", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 1192, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-139", "ok": false, "error": null, "completion_tokens": 236, "reasoning_chars": 769, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_security-22", "ok": true, "error": null, "completion_tokens": 84, "reasoning_chars": 392, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-7", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-108", "ok": true, "error": null, "completion_tokens": 697, "reasoning_chars": 1777, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-31", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-131", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 653, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-35", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 51, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_actuarial_science-38", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-165", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-101", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-107", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 138, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-8", "ok": false, "error": null, "completion_tokens": 3012, "reasoning_chars": 11890, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medical_statistics-86", "ok": true, "error": null, "completion_tokens": 587, "reasoning_chars": 2069, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-security_study-30", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 1363, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-118", "ok": true, "error": null, "completion_tokens": 430, "reasoning_chars": 1697, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_law-24", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-46", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-anatomy-21", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-145", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-91", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 21, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-109", "ok": false, "error": null, "completion_tokens": 44, "reasoning_chars": 51, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-44", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-74", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 56, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-63", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-146", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-109", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 789, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-construction_project_management-16", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 161, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-113", "ok": false, "error": null, "completion_tokens": 242, "reasoning_chars": 978, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-102", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-207", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-19", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-164", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 1198, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-338", "ok": true, "error": null, "completion_tokens": 40, "reasoning_chars": 53, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-7", "ok": true, "error": null, "completion_tokens": 478, "reasoning_chars": 1979, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-92", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-47", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 1511, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-140", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 22, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-100", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 961, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-37", "ok": true, "error": null, "completion_tokens": 614, "reasoning_chars": 1969, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-191", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-34", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 191, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-88", "ok": true, "error": null, "completion_tokens": 421, "reasoning_chars": 1324, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-77", "ok": false, "error": null, "completion_tokens": 602, "reasoning_chars": 2244, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-306", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-40", "ok": true, "error": null, "completion_tokens": 288, "reasoning_chars": 1180, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_chemistry-7", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-152", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 823, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-1", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-94", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 619, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-109", "ok": true, "error": null, "completion_tokens": 192, "reasoning_chars": 827, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-164", "ok": true, "error": null, "completion_tokens": 116, "reasoning_chars": 159, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-89", "ok": false, "error": null, "completion_tokens": 290, "reasoning_chars": 1264, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-17", "ok": true, "error": null, "completion_tokens": 456, "reasoning_chars": 1475, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-2", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 214, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-51", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-141", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 1065, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-196", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 90, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-97", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 61, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-202", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-195", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-49", "ok": true, "error": null, "completion_tokens": 652, "reasoning_chars": 2573, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-94", "ok": true, "error": null, "completion_tokens": 325, "reasoning_chars": 1434, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-8", "ok": false, "error": null, "completion_tokens": 470, "reasoning_chars": 632, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-98", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 277, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-28", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-74", "ok": true, "error": null, "completion_tokens": 839, "reasoning_chars": 2880, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-121", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 1204, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-120", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 481, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-105", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 152, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-35", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 677, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-51", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 897, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-49", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-130", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 1441, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-140", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "mmlu_pro": { "task": "mmlu_pro", "n": 150, "hits": 106, "accuracy_pct": 70.67, "accuracy_ex_trunc_pct": 71.14, "truncated": 1, "completion_tokens_p50": 374, "completion_tokens_p90": 2631, "completion_tokens_max": 32768, "reasoning_chars_p50": 678, "reasoning_chars_p90": 3617, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 949.4, "gen_tokens": 241357, "tok_per_s": 254.2, "details": [ { "id": "mmlupro-5831", "ok": true, "error": null, "completion_tokens": 269, "reasoning_chars": 721, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2299", "ok": true, "error": null, "completion_tokens": 590, "reasoning_chars": 2136, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9280", "ok": false, "error": null, "completion_tokens": 200, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5486", "ok": true, "error": null, "completion_tokens": 762, "reasoning_chars": 3617, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5761", "ok": false, "error": null, "completion_tokens": 116, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6589", "ok": true, "error": null, "completion_tokens": 734, "reasoning_chars": 1904, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5389", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 1336, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5131", "ok": true, "error": null, "completion_tokens": 838, "reasoning_chars": 2111, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1677", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4388", "ok": false, "error": null, "completion_tokens": 28753, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3285", "ok": true, "error": null, "completion_tokens": 935, "reasoning_chars": 2392, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4118", "ok": true, "error": null, "completion_tokens": 1434, "reasoning_chars": 3699, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5344", "ok": false, "error": null, "completion_tokens": 64, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11696", "ok": false, "error": null, "completion_tokens": 4133, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-858", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-214", "ok": false, "error": null, "completion_tokens": 4667, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6609", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2083", "ok": true, "error": null, "completion_tokens": 772, "reasoning_chars": 2871, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9752", "ok": false, "error": null, "completion_tokens": 10261, "reasoning_chars": 22761, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7213", "ok": true, "error": null, "completion_tokens": 500, "reasoning_chars": 1623, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5234", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 594, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8570", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5612", "ok": false, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2071", "ok": true, "error": null, "completion_tokens": 692, "reasoning_chars": 2485, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9842", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4665", "ok": true, "error": null, "completion_tokens": 957, "reasoning_chars": 2120, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5564", "ok": true, "error": null, "completion_tokens": 466, "reasoning_chars": 1974, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6475", "ok": true, "error": null, "completion_tokens": 910, "reasoning_chars": 2549, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2153", "ok": true, "error": null, "completion_tokens": 246, "reasoning_chars": 930, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5812", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 247, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3407", "ok": true, "error": null, "completion_tokens": 2631, "reasoning_chars": 4215, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2879", "ok": false, "error": null, "completion_tokens": 311, "reasoning_chars": 495, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8266", "ok": false, "error": null, "completion_tokens": 54, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1916", "ok": true, "error": null, "completion_tokens": 822, "reasoning_chars": 3127, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3486", "ok": true, "error": null, "completion_tokens": 534, "reasoning_chars": 1943, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11275", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6040", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8565", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 249, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6848", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 303, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8926", "ok": false, "error": null, "completion_tokens": 120, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8458", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 98, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7854", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 1155, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3962", "ok": true, "error": null, "completion_tokens": 7209, "reasoning_chars": 8947, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6470", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3813", "ok": true, "error": null, "completion_tokens": 376, "reasoning_chars": 1012, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8026", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11030", "ok": true, "error": null, "completion_tokens": 693, "reasoning_chars": 2203, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8151", "ok": true, "error": null, "completion_tokens": 40, "reasoning_chars": 15, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5286", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 890, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3888", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11676", "ok": false, "error": null, "completion_tokens": 1016, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8762", "ok": false, "error": null, "completion_tokens": 794, "reasoning_chars": 1515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-12160", "ok": false, "error": null, "completion_tokens": 539, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4022", "ok": true, "error": null, "completion_tokens": 797, "reasoning_chars": 2290, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10092", "ok": true, "error": null, "completion_tokens": 2959, "reasoning_chars": 9541, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7364", "ok": true, "error": null, "completion_tokens": 277, "reasoning_chars": 701, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6001", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10873", "ok": true, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9777", "ok": true, "error": null, "completion_tokens": 2957, "reasoning_chars": 4812, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8723", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4595", "ok": true, "error": null, "completion_tokens": 374, "reasoning_chars": 921, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8423", "ok": false, "error": null, "completion_tokens": 1490, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11051", "ok": false, "error": null, "completion_tokens": 135, "reasoning_chars": 678, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-836", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11297", "ok": true, "error": null, "completion_tokens": 31236, "reasoning_chars": 85173, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3950", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 905, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1562", "ok": false, "error": null, "completion_tokens": 552, "reasoning_chars": 1952, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2041", "ok": true, "error": null, "completion_tokens": 744, "reasoning_chars": 2725, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3473", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11253", "ok": false, "error": null, "completion_tokens": 276, "reasoning_chars": 1283, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3858", "ok": true, "error": null, "completion_tokens": 716, "reasoning_chars": 1262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2231", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2190", "ok": false, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6930", "ok": true, "error": null, "completion_tokens": 489, "reasoning_chars": 1992, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6558", "ok": true, "error": null, "completion_tokens": 782, "reasoning_chars": 2394, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5371", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1766", "ok": true, "error": null, "completion_tokens": 898, "reasoning_chars": 2986, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6045", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2298", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 882, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6400", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6499", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7936", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 1146, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8566", "ok": true, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11633", "ok": true, "error": null, "completion_tokens": 685, "reasoning_chars": 783, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9843", "ok": false, "error": null, "completion_tokens": 4328, "reasoning_chars": 7266, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11758", "ok": false, "error": null, "completion_tokens": 24725, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2843", "ok": true, "error": null, "completion_tokens": 1551, "reasoning_chars": 4524, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7483", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11379", "ok": true, "error": null, "completion_tokens": 3403, "reasoning_chars": 5489, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4866", "ok": true, "error": null, "completion_tokens": 701, "reasoning_chars": 2188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-818", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 155, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2785", "ok": true, "error": null, "completion_tokens": 845, "reasoning_chars": 3191, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8119", "ok": false, "error": null, "completion_tokens": 215, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2214", "ok": true, "error": null, "completion_tokens": 594, "reasoning_chars": 1534, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5774", "ok": true, "error": null, "completion_tokens": 1317, "reasoning_chars": 4871, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6049", "ok": false, "error": null, "completion_tokens": 473, "reasoning_chars": 1485, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7219", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 1399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1660", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4767", "ok": true, "error": null, "completion_tokens": 62, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5964", "ok": false, "error": null, "completion_tokens": 389, "reasoning_chars": 1095, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2743", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4062", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 148, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7368", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-827", "ok": false, "error": null, "completion_tokens": 569, "reasoning_chars": 2414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11877", "ok": false, "error": null, "completion_tokens": 8798, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7557", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9998", "ok": false, "error": null, "completion_tokens": 77, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10098", "ok": false, "error": null, "completion_tokens": 935, "reasoning_chars": 2838, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1744", "ok": false, "error": null, "completion_tokens": 1053, "reasoning_chars": 3786, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9210", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 509, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11248", "ok": false, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11187", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2799", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4479", "ok": true, "error": null, "completion_tokens": 1396, "reasoning_chars": 2703, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10223", "ok": true, "error": null, "completion_tokens": 1292, "reasoning_chars": 1644, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5698", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8291", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 140, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3516", "ok": true, "error": null, "completion_tokens": 781, "reasoning_chars": 2016, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5466", "ok": true, "error": null, "completion_tokens": 916, "reasoning_chars": 2937, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4851", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-733", "ok": true, "error": null, "completion_tokens": 1880, "reasoning_chars": 2793, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6995", "ok": true, "error": null, "completion_tokens": 423, "reasoning_chars": 1231, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2573", "ok": true, "error": null, "completion_tokens": 649, "reasoning_chars": 2767, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8481", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 438, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5938", "ok": true, "error": null, "completion_tokens": 741, "reasoning_chars": 2781, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10708", "ok": false, "error": null, "completion_tokens": 79, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8476", "ok": true, "error": null, "completion_tokens": 445, "reasoning_chars": 1244, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7990", "ok": true, "error": null, "completion_tokens": 302, "reasoning_chars": 961, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2409", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 2235, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11551", "ok": true, "error": null, "completion_tokens": 547, "reasoning_chars": 454, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-674", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 132, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1978", "ok": true, "error": null, "completion_tokens": 1969, "reasoning_chars": 7713, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10420", "ok": false, "error": null, "completion_tokens": 137, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3445", "ok": true, "error": null, "completion_tokens": 981, "reasoning_chars": 3429, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9488", "ok": true, "error": null, "completion_tokens": 655, "reasoning_chars": 657, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4993", "ok": true, "error": null, "completion_tokens": 397, "reasoning_chars": 1364, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4316", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4730", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 807, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11570", "ok": false, "error": null, "completion_tokens": 48, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2149", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 1316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8322", "ok": true, "error": null, "completion_tokens": 8483, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10993", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4588", "ok": false, "error": null, "completion_tokens": 7429, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3258", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8629", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 415, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-637", "ok": true, "error": null, "completion_tokens": 319, "reasoning_chars": 359, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4257", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 799, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9719", "ok": true, "error": null, "completion_tokens": 701, "reasoning_chars": 1810, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11714", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 74659, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlupro-9678", "ok": false, "error": null, "completion_tokens": 1355, "reasoning_chars": 4710, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ceval": { "task": "ceval", "n": 150, "hits": 114, "accuracy_pct": 76.0, "accuracy_ex_trunc_pct": 76.51, "truncated": 1, "completion_tokens_p50": 197, "completion_tokens_p90": 1003, "completion_tokens_max": 32768, "reasoning_chars_p50": 403, "reasoning_chars_p90": 2994, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 1347.6, "gen_tokens": 108193, "tok_per_s": 80.3, "details": [ { "id": "ceval-high_school_chinese-1", "ok": false, "error": null, "completion_tokens": 918, "reasoning_chars": 1439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-10", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-41", "ok": false, "error": null, "completion_tokens": 3944, "reasoning_chars": 10623, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-3", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-21", "ok": true, "error": null, "completion_tokens": 645, "reasoning_chars": 2994, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-4", "ok": true, "error": null, "completion_tokens": 2964, "reasoning_chars": 6989, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-21", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 1029, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-2", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 142, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-6", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-12", "ok": false, "error": null, "completion_tokens": 1598, "reasoning_chars": 5557, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-19", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chemistry-9", "ok": true, "error": null, "completion_tokens": 3947, "reasoning_chars": 14603, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-15", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-22", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-16", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 586, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-11", "ok": false, "error": null, "completion_tokens": 218, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-18", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 158, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-24", "ok": true, "error": null, "completion_tokens": 888, "reasoning_chars": 2759, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-29", "ok": true, "error": null, "completion_tokens": 1573, "reasoning_chars": 5949, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-2", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 504, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-11", "ok": false, "error": null, "completion_tokens": 245, "reasoning_chars": 1109, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-18", "ok": true, "error": null, "completion_tokens": 465, "reasoning_chars": 1808, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-0", "ok": false, "error": null, "completion_tokens": 379, "reasoning_chars": 1397, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-14", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-27", "ok": true, "error": null, "completion_tokens": 659, "reasoning_chars": 2771, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-19", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 277, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-21", "ok": true, "error": null, "completion_tokens": 1072, "reasoning_chars": 3558, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-33", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 403, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-20", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-10", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-19", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-7", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 59, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-41", "ok": true, "error": null, "completion_tokens": 439, "reasoning_chars": 1301, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-44", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 1254, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-9", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 468, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-28", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 1369, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-39", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-1", "ok": false, "error": null, "completion_tokens": 301, "reasoning_chars": 992, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-24", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 612, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-5", "ok": true, "error": null, "completion_tokens": 1616, "reasoning_chars": 5596, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-computer_architecture-15", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-39", "ok": true, "error": null, "completion_tokens": 409, "reasoning_chars": 1914, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-23", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 323, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-7", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 1123, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-17", "ok": true, "error": null, "completion_tokens": 224, "reasoning_chars": 1056, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-6", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 850, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-15", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 112, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-4", "ok": false, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-logic-5", "ok": true, "error": null, "completion_tokens": 811, "reasoning_chars": 3293, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-50", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_politics-11", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-32", "ok": true, "error": null, "completion_tokens": 822, "reasoning_chars": 2600, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-16", "ok": true, "error": null, "completion_tokens": 873, "reasoning_chars": 2367, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-9", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 293, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-15", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 109, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-2", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-7", "ok": true, "error": null, "completion_tokens": 865, "reasoning_chars": 2931, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-11", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-operating_system-18", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 139, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-4", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-17", "ok": true, "error": null, "completion_tokens": 466, "reasoning_chars": 1245, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-1", "ok": true, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-32", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 147, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-33", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-25", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 897, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-40", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-7", "ok": true, "error": null, "completion_tokens": 1176, "reasoning_chars": 4286, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-5", "ok": true, "error": null, "completion_tokens": 367, "reasoning_chars": 1742, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-12", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-4", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 235, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-38", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 44005, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "ceval-electrical_engineer-22", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 693, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-15", "ok": false, "error": null, "completion_tokens": 524, "reasoning_chars": 1723, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-45", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-logic-6", "ok": true, "error": null, "completion_tokens": 609, "reasoning_chars": 889, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-20", "ok": true, "error": null, "completion_tokens": 981, "reasoning_chars": 3145, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-2", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-17", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-16", "ok": false, "error": null, "completion_tokens": 509, "reasoning_chars": 2418, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-7", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 784, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_biology-16", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-10", "ok": false, "error": null, "completion_tokens": 123, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-32", "ok": true, "error": null, "completion_tokens": 7817, "reasoning_chars": 21220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-0", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 134, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-12", "ok": true, "error": null, "completion_tokens": 981, "reasoning_chars": 1710, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-14", "ok": true, "error": null, "completion_tokens": 654, "reasoning_chars": 2626, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-26", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-3", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-3", "ok": true, "error": null, "completion_tokens": 101, "reasoning_chars": 128, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-13", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-5", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-27", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 705, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-10", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 233, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-16", "ok": false, "error": null, "completion_tokens": 37, "reasoning_chars": 36, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-49", "ok": true, "error": null, "completion_tokens": 302, "reasoning_chars": 1219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-13", "ok": false, "error": null, "completion_tokens": 648, "reasoning_chars": 2015, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-5", "ok": true, "error": null, "completion_tokens": 4266, "reasoning_chars": 11569, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-2", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 925, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-27", "ok": false, "error": null, "completion_tokens": 263, "reasoning_chars": 1039, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_biology-8", "ok": false, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-14", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-14", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 508, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-31", "ok": true, "error": null, "completion_tokens": 448, "reasoning_chars": 724, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-13", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 1332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-12", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 1129, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-44", "ok": false, "error": null, "completion_tokens": 1172, "reasoning_chars": 1807, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-3", "ok": true, "error": null, "completion_tokens": 1576, "reasoning_chars": 4636, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-13", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-6", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 78, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_politics-19", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 919, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-environmental_impact_assessment_engineer-27", "ok": false, "error": null, "completion_tokens": 9119, "reasoning_chars": 22990, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-10", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-0", "ok": true, "error": null, "completion_tokens": 381, "reasoning_chars": 1555, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-3", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 192, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-6", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 1903, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chinese-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-43", "ok": false, "error": null, "completion_tokens": 425, "reasoning_chars": 1428, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-42", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 172, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-13", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 878, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-discrete_mathematics-1", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 89, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-11", "ok": false, "error": null, "completion_tokens": 83, "reasoning_chars": 360, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-16", "ok": true, "error": null, "completion_tokens": 1003, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-2", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 777, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-1", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 843, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-16", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-11", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 967, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-5", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 1132, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-40", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 1754, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-6", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-10", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 7, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-10", "ok": true, "error": null, "completion_tokens": 1399, "reasoning_chars": 2522, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-17", "ok": false, "error": null, "completion_tokens": 134, "reasoning_chars": 238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-11", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 1794, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-16", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 121, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-1", "ok": true, "error": null, "completion_tokens": 551, "reasoning_chars": 1473, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-0", "ok": false, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-37", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 1105, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-9", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 35, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_history-16", "ok": true, "error": null, "completion_tokens": 178, "reasoning_chars": 254, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-5", "ok": true, "error": null, "completion_tokens": 27, "reasoning_chars": 23, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-18", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 116, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-30", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-21", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 790, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-42", "ok": true, "error": null, "completion_tokens": 798, "reasoning_chars": 1300, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-5", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-4", "ok": true, "error": null, "completion_tokens": 286, "reasoning_chars": 825, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "arc": { "task": "arc", "n": 150, "hits": 128, "accuracy_pct": 85.33, "accuracy_ex_trunc_pct": 85.33, "truncated": 0, "completion_tokens_p50": 244, "completion_tokens_p90": 446, "completion_tokens_max": 1431, "reasoning_chars_p50": 730, "reasoning_chars_p90": 1388, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1331.1, "gen_tokens": 38806, "tok_per_s": 29.2, "details": [ { "id": "arc-Mercury_415084", "ok": true, "error": null, "completion_tokens": 48, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7249708", "ok": true, "error": null, "completion_tokens": 442, "reasoning_chars": 1325, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_4_24", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 858, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7056823", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 710, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415534", "ok": true, "error": null, "completion_tokens": 178, "reasoning_chars": 419, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405827", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 1053, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263183", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 559, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106960", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 195, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_7", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7182245", "ok": true, "error": null, "completion_tokens": 453, "reasoning_chars": 1611, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416177", "ok": false, "error": null, "completion_tokens": 101, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2011_5_8", "ok": true, "error": null, "completion_tokens": 387, "reasoning_chars": 1095, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_8_36", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 127, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401699", "ok": true, "error": null, "completion_tokens": 101, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107240", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 839, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_4_26", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 791, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402638", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 1090, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026723", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 155, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415422", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 959, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2000_8_2", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 821, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2006_9_4", "ok": false, "error": null, "completion_tokens": 81, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401272", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 392, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416169", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 97, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7116235", "ok": true, "error": null, "completion_tokens": 467, "reasoning_chars": 1394, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7109463", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 717, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7032428", "ok": false, "error": null, "completion_tokens": 593, "reasoning_chars": 1906, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2008_8_11", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 1126, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_410334", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 1008, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_36", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 401, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415719", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_17", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 700, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402256", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 670, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_400443", "ok": true, "error": null, "completion_tokens": 217, "reasoning_chars": 311, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7133245", "ok": true, "error": null, "completion_tokens": 336, "reasoning_chars": 1171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413458", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401372", "ok": true, "error": null, "completion_tokens": 259, "reasoning_chars": 884, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401116", "ok": false, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__5_10312", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 416, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2007_4_10280", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416683", "ok": true, "error": null, "completion_tokens": 543, "reasoning_chars": 1672, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7190120", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 571, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401295", "ok": true, "error": null, "completion_tokens": 373, "reasoning_chars": 1439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_4_28", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 287, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7223493", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 877, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018410", "ok": false, "error": null, "completion_tokens": 92, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1998_8_24", "ok": false, "error": null, "completion_tokens": 325, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_31", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405501", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 878, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408030", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401684", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 763, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041668", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 850, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7205870", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 909, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7210140", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 1108, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405873", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 1431, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400052", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 683, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018270", "ok": true, "error": null, "completion_tokens": 390, "reasoning_chars": 1423, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406550", "ok": true, "error": null, "completion_tokens": 347, "reasoning_chars": 844, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7150728", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 1276, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400839", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 859, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_30", "ok": true, "error": null, "completion_tokens": 77, "reasoning_chars": 298, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400383", "ok": true, "error": null, "completion_tokens": 500, "reasoning_chars": 1351, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7236740", "ok": true, "error": null, "completion_tokens": 170, "reasoning_chars": 263, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-CSZ_2005_5_CSZ10021", "ok": false, "error": null, "completion_tokens": 217, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7093958", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 867, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026355", "ok": true, "error": null, "completion_tokens": 485, "reasoning_chars": 1448, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408324", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 87, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__7_10348", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 413, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402629", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 193, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400046", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 1018, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7180810", "ok": true, "error": null, "completion_tokens": 554, "reasoning_chars": 1748, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_180005", "ok": false, "error": null, "completion_tokens": 86, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2009_8_14", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 1271, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2007_5_16", "ok": true, "error": null, "completion_tokens": 101, "reasoning_chars": 419, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_LBS10662", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 987, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_23", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 703, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2012_8_19", "ok": false, "error": null, "completion_tokens": 163, "reasoning_chars": 504, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041650", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 887, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416181", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 288, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_1995_8_J6", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 124, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2014_5_13", "ok": true, "error": null, "completion_tokens": 332, "reasoning_chars": 1015, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106750", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 973, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_4_23", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 552, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MEA_2013_5_12", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 150, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_8_42", "ok": false, "error": null, "completion_tokens": 106, "reasoning_chars": 342, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408367", "ok": false, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7250058", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 874, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_414041", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 1139, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401290", "ok": false, "error": null, "completion_tokens": 246, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409295", "ok": true, "error": null, "completion_tokens": 577, "reasoning_chars": 1651, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7211680", "ok": true, "error": null, "completion_tokens": 467, "reasoning_chars": 1388, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_37", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 1067, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413439", "ok": true, "error": null, "completion_tokens": 428, "reasoning_chars": 1211, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_32", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 805, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7216423", "ok": true, "error": null, "completion_tokens": 343, "reasoning_chars": 1047, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2015_7_9", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 203, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7230405", "ok": true, "error": null, "completion_tokens": 573, "reasoning_chars": 1556, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7025008", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 1044, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416536", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 426, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_2007_8_pg60", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 492, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_4_34", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 881, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_406833", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 265, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409647", "ok": true, "error": null, "completion_tokens": 494, "reasoning_chars": 1389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_184363", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 754, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107170", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 662, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416104", "ok": false, "error": null, "completion_tokens": 94, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_10-v1", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7202843", "ok": true, "error": null, "completion_tokens": 472, "reasoning_chars": 1338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029803", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 1132, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7164623", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 1064, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_26", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 442, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263060", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 1183, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268328", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 884, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402282", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 670, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029558", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 174, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_4_21", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7186358", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 1344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2011_8_10434", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 1191, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2012_8_23651", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 851, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_407125", "ok": true, "error": null, "completion_tokens": 593, "reasoning_chars": 2020, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268258", "ok": false, "error": null, "completion_tokens": 56, "reasoning_chars": 244, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7027405", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 826, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_8_7", "ok": true, "error": null, "completion_tokens": 1431, "reasoning_chars": 6785, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_191625", "ok": false, "error": null, "completion_tokens": 269, "reasoning_chars": 1057, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_6", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 237, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7219678", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 1188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401678", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 766, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401312", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 376, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_8_42", "ok": false, "error": null, "completion_tokens": 203, "reasoning_chars": 149, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7030468", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 1136, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175385", "ok": true, "error": null, "completion_tokens": 419, "reasoning_chars": 911, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175560", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_5", "ok": true, "error": null, "completion_tokens": 53, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_409009", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 929, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7128923", "ok": false, "error": null, "completion_tokens": 319, "reasoning_chars": 753, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_8_9", "ok": false, "error": null, "completion_tokens": 185, "reasoning_chars": 642, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7042945", "ok": false, "error": null, "completion_tokens": 233, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_8_47", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 243, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7185343", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7218400", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 534, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7221988", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 522, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405950", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 1156, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7222530", "ok": true, "error": null, "completion_tokens": 417, "reasoning_chars": 1246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7102323", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_5_39", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 589, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_8_5", "ok": true, "error": null, "completion_tokens": 289, "reasoning_chars": 730, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400661", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 776, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7081603", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 1584, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2008_3_25", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 388, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413628", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 499, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406427", "ok": true, "error": null, "completion_tokens": 312, "reasoning_chars": 1119, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "truthfulqa": { "task": "truthfulqa", "n": 150, "hits": 107, "accuracy_pct": 71.33, "accuracy_ex_trunc_pct": 71.33, "truncated": 0, "completion_tokens_p50": 65, "completion_tokens_p90": 510, "completion_tokens_max": 994, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1579, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1294.4, "gen_tokens": 27440, "tok_per_s": 21.2, "details": [ { "id": "tqa-210", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-76", "ok": false, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-461", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 1814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-218", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-254", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-635", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-193", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-148", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 1411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-400", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-766", "ok": true, "error": null, "completion_tokens": 537, "reasoning_chars": 1896, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-240", "ok": false, "error": null, "completion_tokens": 666, "reasoning_chars": 2396, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-50", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-83", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-381", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-325", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-655", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-721", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-391", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-401", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 1005, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-625", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 1053, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-80", "ok": false, "error": null, "completion_tokens": 289, "reasoning_chars": 751, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-439", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-66", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 1572, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-155", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-516", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-522", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-47", "ok": true, "error": null, "completion_tokens": 416, "reasoning_chars": 1417, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-343", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-126", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-77", "ok": true, "error": null, "completion_tokens": 550, "reasoning_chars": 1579, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-613", "ok": false, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-590", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-674", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-438", "ok": false, "error": null, "completion_tokens": 186, "reasoning_chars": 696, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-503", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-639", "ok": true, "error": null, "completion_tokens": 76, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-111", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-289", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-705", "ok": false, "error": null, "completion_tokens": 849, "reasoning_chars": 3294, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-373", "ok": false, "error": null, "completion_tokens": 201, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-363", "ok": false, "error": null, "completion_tokens": 469, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-792", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-113", "ok": false, "error": null, "completion_tokens": 579, "reasoning_chars": 1762, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-74", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 759, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-634", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-804", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-626", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-676", "ok": true, "error": null, "completion_tokens": 261, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-174", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 638, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-455", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-725", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-296", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-535", "ok": true, "error": null, "completion_tokens": 332, "reasoning_chars": 1335, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-172", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-23", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-43", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-38", "ok": false, "error": null, "completion_tokens": 196, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-97", "ok": false, "error": null, "completion_tokens": 254, "reasoning_chars": 726, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-657", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-814", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-346", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 1420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-365", "ok": false, "error": null, "completion_tokens": 136, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-384", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-677", "ok": false, "error": null, "completion_tokens": 646, "reasoning_chars": 2114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-486", "ok": false, "error": null, "completion_tokens": 86, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-55", "ok": true, "error": null, "completion_tokens": 62, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-680", "ok": true, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-510", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-128", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-805", "ok": false, "error": null, "completion_tokens": 260, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-394", "ok": false, "error": null, "completion_tokens": 432, "reasoning_chars": 1679, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-435", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-285", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-687", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-588", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-100", "ok": true, "error": null, "completion_tokens": 338, "reasoning_chars": 1315, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-399", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 355, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-266", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-693", "ok": true, "error": null, "completion_tokens": 640, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-532", "ok": false, "error": null, "completion_tokens": 344, "reasoning_chars": 1100, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-558", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 768, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-319", "ok": true, "error": null, "completion_tokens": 333, "reasoning_chars": 1090, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-459", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-675", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 843, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-129", "ok": true, "error": null, "completion_tokens": 994, "reasoning_chars": 3054, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-331", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-12", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 752, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-164", "ok": false, "error": null, "completion_tokens": 982, "reasoning_chars": 3135, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-306", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 362, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-275", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-418", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-264", "ok": false, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-94", "ok": true, "error": null, "completion_tokens": 372, "reasoning_chars": 1171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-389", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-427", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 878, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-269", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-643", "ok": false, "error": null, "completion_tokens": 228, "reasoning_chars": 658, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-7", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-419", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-616", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 388, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-699", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 1130, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-473", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-523", "ok": false, "error": null, "completion_tokens": 289, "reasoning_chars": 853, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-214", "ok": true, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-376", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-123", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 434, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-200", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 560, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-493", "ok": true, "error": null, "completion_tokens": 353, "reasoning_chars": 1304, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-474", "ok": true, "error": null, "completion_tokens": 101, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-387", "ok": false, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-746", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 422, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-790", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-181", "ok": false, "error": null, "completion_tokens": 879, "reasoning_chars": 3124, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-413", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-349", "ok": true, "error": null, "completion_tokens": 510, "reasoning_chars": 2077, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-434", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 506, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-601", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 1054, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-489", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 317, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-286", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-488", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-759", "ok": false, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-273", "ok": true, "error": null, "completion_tokens": 390, "reasoning_chars": 1200, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-59", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-315", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-425", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-58", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-691", "ok": true, "error": null, "completion_tokens": 393, "reasoning_chars": 1217, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-497", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-515", "ok": true, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-398", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-670", "ok": true, "error": null, "completion_tokens": 873, "reasoning_chars": 1722, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-170", "ok": false, "error": null, "completion_tokens": 504, "reasoning_chars": 1484, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-334", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-307", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-495", "ok": true, "error": null, "completion_tokens": 973, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-597", "ok": true, "error": null, "completion_tokens": 774, "reasoning_chars": 1832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-775", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-186", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-733", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-308", "ok": false, "error": null, "completion_tokens": 574, "reasoning_chars": 1895, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-5", "ok": true, "error": null, "completion_tokens": 584, "reasoning_chars": 1899, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-231", "ok": false, "error": null, "completion_tokens": 280, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-356", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-810", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-270", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-13", "ok": false, "error": null, "completion_tokens": 167, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-114", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 430, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-4", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-417", "ok": true, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-750", "ok": false, "error": null, "completion_tokens": 65, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "gsm8k": { "task": "gsm8k", "n": 100, "hits": 88, "accuracy_pct": 88.0, "accuracy_ex_trunc_pct": 88.0, "truncated": 0, "completion_tokens_p50": 139, "completion_tokens_p90": 323, "completion_tokens_max": 581, "reasoning_chars_p50": 197, "reasoning_chars_p90": 702, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1154.1, "gen_tokens": 15118, "tok_per_s": 13.1, "details": [ { "id": "gsm8k-144", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 944, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-627", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 778, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-195", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-201", "ok": false, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-730", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 267, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-166", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 274, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1141", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-832", "ok": false, "error": null, "completion_tokens": 54, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-53", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 425, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1042", "ok": false, "error": null, "completion_tokens": 77, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-609", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 299, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-318", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-892", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 381, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-204", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1120", "ok": false, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1040", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 702, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-556", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1015", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 311, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-206", "ok": true, "error": null, "completion_tokens": 328, "reasoning_chars": 754, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1058", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1044", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-131", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1154", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 145, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-158", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1149", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 284, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1156", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 388, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-60", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-918", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1010", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1032", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1053", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-577", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 301, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-935", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-692", "ok": true, "error": null, "completion_tokens": 84, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-153", "ok": true, "error": null, "completion_tokens": 518, "reasoning_chars": 1318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-40", "ok": true, "error": null, "completion_tokens": 486, "reasoning_chars": 1494, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-613", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-875", "ok": true, "error": null, "completion_tokens": 145, "reasoning_chars": 239, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-189", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-800", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-169", "ok": true, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1169", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-211", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 410, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1183", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-193", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 759, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-230", "ok": true, "error": null, "completion_tokens": 329, "reasoning_chars": 467, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-787", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 346, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-83", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1318", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 261, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-733", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 86, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1205", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 390, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-88", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 266, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-509", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-311", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 292, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-874", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-287", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 329, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1055", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1246", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 324, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-975", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-280", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-541", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-974", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 253, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-827", "ok": true, "error": null, "completion_tokens": 448, "reasoning_chars": 1064, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-125", "ok": true, "error": null, "completion_tokens": 581, "reasoning_chars": 1831, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-453", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-942", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1199", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-884", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-855", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 520, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1270", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-349", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1231", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 298, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1106", "ok": true, "error": null, "completion_tokens": 101, "reasoning_chars": 253, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-803", "ok": true, "error": null, "completion_tokens": 218, "reasoning_chars": 395, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-922", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-152", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 291, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-920", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-207", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 265, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-297", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 345, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-786", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 165, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-101", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 421, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1301", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-27", "ok": true, "error": null, "completion_tokens": 84, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1005", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-659", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 450, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-340", "ok": false, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-288", "ok": true, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-763", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 408, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-840", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-742", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 187, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-222", "ok": true, "error": null, "completion_tokens": 51, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-468", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-220", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 197, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-662", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-269", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1193", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 559, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-270", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-114", "ok": true, "error": null, "completion_tokens": 426, "reasoning_chars": 1220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-728", "ok": true, "error": null, "completion_tokens": 228, "reasoning_chars": 371, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-396", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "math500": { "task": "math500", "n": 100, "hits": 73, "accuracy_pct": 73.0, "accuracy_ex_trunc_pct": 74.49, "truncated": 2, "completion_tokens_p50": 395, "completion_tokens_p90": 2854, "completion_tokens_max": 32768, "reasoning_chars_p50": 702, "reasoning_chars_p90": 6278, "reasoning_leaked": null, "empty": 2, "length_limited": 2, "errors": 0, "elapsed_s": 1933.2, "gen_tokens": 176664, "tok_per_s": 91.4, "details": [ { "id": "math-test/algebra/1553.json", "ok": true, "error": null, "completion_tokens": 486, "reasoning_chars": 704, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/538.json", "ok": true, "error": null, "completion_tokens": 707, "reasoning_chars": 1236, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/691.json", "ok": false, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1090.json", "ok": true, "error": null, "completion_tokens": 768, "reasoning_chars": 1667, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/116.json", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 346, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1255.json", "ok": true, "error": null, "completion_tokens": 943, "reasoning_chars": 2763, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1143.json", "ok": true, "error": null, "completion_tokens": 762, "reasoning_chars": 1380, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1201.json", "ok": true, "error": null, "completion_tokens": 1503, "reasoning_chars": 2871, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/964.json", "ok": true, "error": null, "completion_tokens": 4249, "reasoning_chars": 7473, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/662.json", "ok": false, "error": null, "completion_tokens": 2753, "reasoning_chars": 4536, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/153.json", "ok": false, "error": null, "completion_tokens": 209, "reasoning_chars": 294, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/276.json", "ok": true, "error": null, "completion_tokens": 473, "reasoning_chars": 657, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1840.json", "ok": true, "error": null, "completion_tokens": 395, "reasoning_chars": 829, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1072.json", "ok": false, "error": null, "completion_tokens": 592, "reasoning_chars": 680, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1907.json", "ok": true, "error": null, "completion_tokens": 137, "reasoning_chars": 234, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/567.json", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 426, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1922.json", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 253, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1779.json", "ok": true, "error": null, "completion_tokens": 5308, "reasoning_chars": 13199, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/951.json", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 423, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1197.json", "ok": false, "error": null, "completion_tokens": 5955, "reasoning_chars": 10312, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/853.json", "ok": true, "error": null, "completion_tokens": 763, "reasoning_chars": 1428, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/516.json", "ok": true, "error": null, "completion_tokens": 717, "reasoning_chars": 1332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1114.json", "ok": false, "error": null, "completion_tokens": 334, "reasoning_chars": 870, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1065.json", "ok": true, "error": null, "completion_tokens": 536, "reasoning_chars": 668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1232.json", "ok": false, "error": null, "completion_tokens": 362, "reasoning_chars": 534, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/769.json", "ok": true, "error": null, "completion_tokens": 593, "reasoning_chars": 689, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1791.json", "ok": false, "error": null, "completion_tokens": 1183, "reasoning_chars": 2477, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/246.json", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 522, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1166.json", "ok": true, "error": null, "completion_tokens": 10658, "reasoning_chars": 20437, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1410.json", "ok": false, "error": null, "completion_tokens": 316, "reasoning_chars": 569, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1686.json", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 449, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/754.json", "ok": false, "error": null, "completion_tokens": 794, "reasoning_chars": 1527, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/477.json", "ok": true, "error": null, "completion_tokens": 492, "reasoning_chars": 1758, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2680.json", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 324, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/23957.json", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 239, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1865.json", "ok": true, "error": null, "completion_tokens": 1257, "reasoning_chars": 2526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1014.json", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 543, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/431.json", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 279, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1425.json", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2584.json", "ok": false, "error": null, "completion_tokens": 229, "reasoning_chars": 320, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2470.json", "ok": false, "error": null, "completion_tokens": 779, "reasoning_chars": 2275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/547.json", "ok": true, "error": null, "completion_tokens": 2854, "reasoning_chars": 8471, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1105.json", "ok": false, "error": null, "completion_tokens": 114, "reasoning_chars": 252, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/347.json", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 829, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1257.json", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 426, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1126.json", "ok": true, "error": null, "completion_tokens": 500, "reasoning_chars": 788, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/239.json", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 194, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1961.json", "ok": false, "error": null, "completion_tokens": 407, "reasoning_chars": 1131, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1388.json", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1078.json", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 961, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2157.json", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/339.json", "ok": true, "error": null, "completion_tokens": 444, "reasoning_chars": 1653, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1761.json", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 369, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1960.json", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 200, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2086.json", "ok": true, "error": null, "completion_tokens": 335, "reasoning_chars": 624, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1462.json", "ok": true, "error": null, "completion_tokens": 3652, "reasoning_chars": 6278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1060.json", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 446, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1055.json", "ok": true, "error": null, "completion_tokens": 1942, "reasoning_chars": 5031, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2517.json", "ok": false, "error": null, "completion_tokens": 654, "reasoning_chars": 1246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/996.json", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 425, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/893.json", "ok": true, "error": null, "completion_tokens": 541, "reasoning_chars": 607, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/51.json", "ok": false, "error": null, "completion_tokens": 336, "reasoning_chars": 494, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1291.json", "ok": true, "error": null, "completion_tokens": 445, "reasoning_chars": 666, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/694.json", "ok": true, "error": null, "completion_tokens": 460, "reasoning_chars": 804, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/711.json", "ok": false, "error": null, "completion_tokens": 23809, "reasoning_chars": 75814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1655.json", "ok": false, "error": null, "completion_tokens": 287, "reasoning_chars": 366, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1936.json", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 324, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1572.json", "ok": true, "error": null, "completion_tokens": 263, "reasoning_chars": 640, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/509.json", "ok": false, "error": null, "completion_tokens": 473, "reasoning_chars": 764, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/864.json", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 260, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1405.json", "ok": true, "error": null, "completion_tokens": 1290, "reasoning_chars": 2746, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/192.json", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 193, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2102.json", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 511, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/843.json", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 717, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/187.json", "ok": true, "error": null, "completion_tokens": 336, "reasoning_chars": 911, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/612.json", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 1093, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/615.json", "ok": true, "error": null, "completion_tokens": 1134, "reasoning_chars": 2988, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1908.json", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 478, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/928.json", "ok": true, "error": null, "completion_tokens": 797, "reasoning_chars": 2010, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/505.json", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1930.json", "ok": false, "error": null, "completion_tokens": 4077, "reasoning_chars": 8434, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1934.json", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1139.json", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 38718, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "math-test/intermediate_algebra/1000.json", "ok": true, "error": null, "completion_tokens": 1175, "reasoning_chars": 2537, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/183.json", "ok": true, "error": null, "completion_tokens": 932, "reasoning_chars": 1651, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1275.json", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2700.json", "ok": false, "error": null, "completion_tokens": 1626, "reasoning_chars": 4639, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2257.json", "ok": false, "error": null, "completion_tokens": 407, "reasoning_chars": 702, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/14.json", "ok": false, "error": null, "completion_tokens": 352, "reasoning_chars": 1118, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1300.json", "ok": false, "error": null, "completion_tokens": 865, "reasoning_chars": 1362, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/515.json", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1423.json", "ok": false, "error": null, "completion_tokens": 334, "reasoning_chars": 808, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1102.json", "ok": true, "error": null, "completion_tokens": 1013, "reasoning_chars": 1965, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1646.json", "ok": true, "error": null, "completion_tokens": 6388, "reasoning_chars": 16612, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1272.json", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/25149.json", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 436, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/697.json", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 33203, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "math-test/algebra/2036.json", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 319, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2037.json", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/353.json", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 426, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "bbh": { "task": "bbh", "n": 150, "hits": 98, "accuracy_pct": 65.33, "accuracy_ex_trunc_pct": 65.1, "truncated": 1, "completion_tokens_p50": 98, "completion_tokens_p90": 559, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1316, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 1319.4, "gen_tokens": 70002, "tok_per_s": 53.1, "details": [ { "id": "bbh-sports_understanding-101", "ok": false, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-60", "ok": true, "error": null, "completion_tokens": 383, "reasoning_chars": 1163, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-113", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-31", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-144", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-28", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 1218, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-116", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-67", "ok": false, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-3", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 665, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-29", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 746, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-192", "ok": true, "error": null, "completion_tokens": 1543, "reasoning_chars": 2947, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-196", "ok": false, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-10", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 321, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-83", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-76", "ok": true, "error": null, "completion_tokens": 784, "reasoning_chars": 2535, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-212", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 824, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-87", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 787, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-1", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 1063, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-140", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-8", "ok": true, "error": null, "completion_tokens": 794, "reasoning_chars": 942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-138", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 1257, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-221", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-136", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-14", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 92, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-0", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 187, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-207", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-32", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-83", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 726, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-220", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-234", "ok": true, "error": null, "completion_tokens": 788, "reasoning_chars": 2685, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-61", "ok": true, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-21", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 495, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-31", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 1030, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-13", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-211", "ok": false, "error": null, "completion_tokens": 220, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-37", "ok": true, "error": null, "completion_tokens": 113, "reasoning_chars": 374, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-225", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-90", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-101", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-179", "ok": false, "error": null, "completion_tokens": 98, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-173", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 61, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-105", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-88", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 558, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-68", "ok": false, "error": null, "completion_tokens": 649, "reasoning_chars": 2480, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-185", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-227", "ok": false, "error": null, "completion_tokens": 128, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-129", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-238", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 738, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-202", "ok": true, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-93", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 842, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-93", "ok": false, "error": null, "completion_tokens": 149, "reasoning_chars": 565, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-161", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-46", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-227", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-158", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-205", "ok": false, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-89", "ok": true, "error": null, "completion_tokens": 76, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-177", "ok": true, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-65", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 549, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-72", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 284, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-49", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 265, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-54", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-64", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 1178, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-86", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-108", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-194", "ok": true, "error": null, "completion_tokens": 337, "reasoning_chars": 738, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-194", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-75", "ok": true, "error": null, "completion_tokens": 4563, "reasoning_chars": 8698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-106", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 298, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-81", "ok": true, "error": null, "completion_tokens": 466, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-242", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-150", "ok": false, "error": null, "completion_tokens": 259, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-25", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 17, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-139", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-12", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 1129, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-178", "ok": true, "error": null, "completion_tokens": 204, "reasoning_chars": 575, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-160", "ok": false, "error": null, "completion_tokens": 343, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-115", "ok": true, "error": null, "completion_tokens": 1835, "reasoning_chars": 7702, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-44", "ok": true, "error": null, "completion_tokens": 556, "reasoning_chars": 466, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-189", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-102", "ok": true, "error": null, "completion_tokens": 513, "reasoning_chars": 760, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-38", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-141", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 1092, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-38", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 296, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-168", "ok": true, "error": null, "completion_tokens": 426, "reasoning_chars": 769, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-212", "ok": true, "error": null, "completion_tokens": 32768, "reasoning_chars": 142106, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "bbh-navigate-245", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-222", "ok": true, "error": null, "completion_tokens": 996, "reasoning_chars": 2516, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-244", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 835, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-35", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 70, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-126", "ok": true, "error": null, "completion_tokens": 519, "reasoning_chars": 1727, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-90", "ok": true, "error": null, "completion_tokens": 850, "reasoning_chars": 2976, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-5", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 764, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-241", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-234", "ok": true, "error": null, "completion_tokens": 559, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-233", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 189, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-5", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-152", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-9", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-209", "ok": false, "error": null, "completion_tokens": 55, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-171", "ok": true, "error": null, "completion_tokens": 451, "reasoning_chars": 1316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-160", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-166", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-213", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-240", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 77, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-95", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-227", "ok": true, "error": null, "completion_tokens": 526, "reasoning_chars": 1672, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-96", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-33", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-4", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-180", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-74", "ok": true, "error": null, "completion_tokens": 178, "reasoning_chars": 245, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-88", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-10", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-191", "ok": false, "error": null, "completion_tokens": 2330, "reasoning_chars": 9530, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-71", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-150", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-188", "ok": true, "error": null, "completion_tokens": 471, "reasoning_chars": 646, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-191", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-203", "ok": false, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-237", "ok": true, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-24", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 158, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-27", "ok": true, "error": null, "completion_tokens": 793, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-177", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-192", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-242", "ok": true, "error": null, "completion_tokens": 380, "reasoning_chars": 905, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-51", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-96", "ok": false, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-100", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 443, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-204", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-58", "ok": true, "error": null, "completion_tokens": 649, "reasoning_chars": 2203, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-157", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 39, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-104", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-47", "ok": false, "error": null, "completion_tokens": 92, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-150", "ok": true, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-23", "ok": true, "error": null, "completion_tokens": 302, "reasoning_chars": 671, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-68", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-174", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-108", "ok": true, "error": null, "completion_tokens": 892, "reasoning_chars": 1596, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-203", "ok": false, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-115", "ok": true, "error": null, "completion_tokens": 428, "reasoning_chars": 733, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-223", "ok": true, "error": null, "completion_tokens": 64, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-221", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-5", "ok": false, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-233", "ok": true, "error": null, "completion_tokens": 257, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-3", "ok": true, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-119", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-197", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-103", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-178", "ok": false, "error": null, "completion_tokens": 915, "reasoning_chars": 2949, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "humaneval": { "task": "humaneval", "n": 164, "hits": 147, "accuracy_pct": 89.63, "accuracy_ex_trunc_pct": 90.18, "truncated": 1, "completion_tokens_p50": 564, "completion_tokens_p90": 5191, "completion_tokens_max": 32768, "reasoning_chars_p50": 1298, "reasoning_chars_p90": 6555, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 2072.3, "gen_tokens": 337798, "tok_per_s": 163.0, "details": [ { "id": "HumanEval/0", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 396, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/1", "ok": true, "error": null, "completion_tokens": 876, "reasoning_chars": 3198, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/2", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 1066, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/3", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 1992, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/4", "ok": true, "error": null, "completion_tokens": 571, "reasoning_chars": 1742, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/5", "ok": false, "error": null, "completion_tokens": 1044, "reasoning_chars": 4669, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/6", "ok": true, "error": null, "completion_tokens": 711, "reasoning_chars": 2572, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/7", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/8", "ok": true, "error": null, "completion_tokens": 380, "reasoning_chars": 843, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/9", "ok": true, "error": null, "completion_tokens": 481, "reasoning_chars": 1918, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/10", "ok": true, "error": null, "completion_tokens": 5675, "reasoning_chars": 19404, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/11", "ok": true, "error": null, "completion_tokens": 10780, "reasoning_chars": 38622, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/12", "ok": true, "error": null, "completion_tokens": 433, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/13", "ok": false, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/14", "ok": true, "error": null, "completion_tokens": 429, "reasoning_chars": 1699, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/15", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 928, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/16", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 605, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/17", "ok": true, "error": null, "completion_tokens": 436, "reasoning_chars": 787, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/18", "ok": true, "error": null, "completion_tokens": 376, "reasoning_chars": 857, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/19", "ok": true, "error": null, "completion_tokens": 565, "reasoning_chars": 1801, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/20", "ok": false, "error": null, "completion_tokens": 1386, "reasoning_chars": 5543, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/21", "ok": true, "error": null, "completion_tokens": 790, "reasoning_chars": 2724, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/22", "ok": false, "error": null, "completion_tokens": 1879, "reasoning_chars": 7069, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/23", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 590, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/24", "ok": true, "error": null, "completion_tokens": 1482, "reasoning_chars": 4310, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/25", "ok": true, "error": null, "completion_tokens": 2996, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/26", "ok": true, "error": null, "completion_tokens": 407, "reasoning_chars": 1192, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/27", "ok": true, "error": null, "completion_tokens": 236, "reasoning_chars": 681, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/28", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 228, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/29", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 630, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/30", "ok": true, "error": null, "completion_tokens": 209, "reasoning_chars": 332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/31", "ok": true, "error": null, "completion_tokens": 1041, "reasoning_chars": 2107, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/32", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 97414, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/33", "ok": true, "error": null, "completion_tokens": 729, "reasoning_chars": 2363, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/34", "ok": false, "error": null, "completion_tokens": 61, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/35", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 354, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/36", "ok": true, "error": null, "completion_tokens": 1445, "reasoning_chars": 3414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/37", "ok": true, "error": null, "completion_tokens": 613, "reasoning_chars": 1518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/38", "ok": false, "error": null, "completion_tokens": 453, "reasoning_chars": 1153, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/39", "ok": true, "error": null, "completion_tokens": 2951, "reasoning_chars": 7413, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/40", "ok": true, "error": null, "completion_tokens": 1638, "reasoning_chars": 4686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/41", "ok": true, "error": null, "completion_tokens": 405, "reasoning_chars": 1059, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/42", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 540, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/43", "ok": true, "error": null, "completion_tokens": 541, "reasoning_chars": 1386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/44", "ok": true, "error": null, "completion_tokens": 6410, "reasoning_chars": 20735, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/45", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 370, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/46", "ok": true, "error": null, "completion_tokens": 1305, "reasoning_chars": 2851, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/47", "ok": false, "error": null, "completion_tokens": 8767, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/48", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 726, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/49", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 930, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/50", "ok": false, "error": null, "completion_tokens": 439, "reasoning_chars": 1276, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/51", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 829, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/52", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 1050, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/53", "ok": false, "error": null, "completion_tokens": 51, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/54", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 796, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/55", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 810, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/56", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 938, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/57", "ok": true, "error": null, "completion_tokens": 509, "reasoning_chars": 1411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/58", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 514, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/59", "ok": true, "error": null, "completion_tokens": 1330, "reasoning_chars": 3467, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/60", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 475, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/61", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 617, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/62", "ok": true, "error": null, "completion_tokens": 363, "reasoning_chars": 906, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/63", "ok": true, "error": null, "completion_tokens": 537, "reasoning_chars": 973, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/64", "ok": true, "error": null, "completion_tokens": 627, "reasoning_chars": 1748, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/65", "ok": true, "error": null, "completion_tokens": 2079, "reasoning_chars": 6555, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/66", "ok": true, "error": null, "completion_tokens": 390, "reasoning_chars": 876, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/67", "ok": false, "error": null, "completion_tokens": 511, "reasoning_chars": 1010, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/68", "ok": true, "error": null, "completion_tokens": 522, "reasoning_chars": 1861, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/69", "ok": true, "error": null, "completion_tokens": 685, "reasoning_chars": 2218, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/70", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 1365, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/71", "ok": true, "error": null, "completion_tokens": 1602, "reasoning_chars": 4466, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/72", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 489, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/73", "ok": true, "error": null, "completion_tokens": 472, "reasoning_chars": 1003, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/74", "ok": true, "error": null, "completion_tokens": 450, "reasoning_chars": 842, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/75", "ok": true, "error": null, "completion_tokens": 5191, "reasoning_chars": 16414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/76", "ok": true, "error": null, "completion_tokens": 12701, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/77", "ok": true, "error": null, "completion_tokens": 1331, "reasoning_chars": 3764, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/78", "ok": true, "error": null, "completion_tokens": 517, "reasoning_chars": 642, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/79", "ok": true, "error": null, "completion_tokens": 491, "reasoning_chars": 1111, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/80", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 1074, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/81", "ok": true, "error": null, "completion_tokens": 1729, "reasoning_chars": 3219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/82", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 764, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/83", "ok": true, "error": null, "completion_tokens": 828, "reasoning_chars": 2099, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/84", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 536, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/85", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 631, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/86", "ok": true, "error": null, "completion_tokens": 816, "reasoning_chars": 2360, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/87", "ok": true, "error": null, "completion_tokens": 759, "reasoning_chars": 1677, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/88", "ok": true, "error": null, "completion_tokens": 600, "reasoning_chars": 1332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/89", "ok": true, "error": null, "completion_tokens": 1006, "reasoning_chars": 2898, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/90", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 1488, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/91", "ok": true, "error": null, "completion_tokens": 9380, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/92", "ok": true, "error": null, "completion_tokens": 518, "reasoning_chars": 1777, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/93", "ok": true, "error": null, "completion_tokens": 1323, "reasoning_chars": 4021, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/94", "ok": true, "error": null, "completion_tokens": 1325, "reasoning_chars": 2862, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/95", "ok": true, "error": null, "completion_tokens": 953, "reasoning_chars": 2692, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/96", "ok": true, "error": null, "completion_tokens": 712, "reasoning_chars": 1477, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/97", "ok": true, "error": null, "completion_tokens": 288, "reasoning_chars": 590, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/98", "ok": true, "error": null, "completion_tokens": 278, "reasoning_chars": 572, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/99", "ok": false, "error": null, "completion_tokens": 2227, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/100", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 914, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/101", "ok": true, "error": null, "completion_tokens": 312, "reasoning_chars": 850, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/102", "ok": true, "error": null, "completion_tokens": 6380, "reasoning_chars": 18934, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/103", "ok": true, "error": null, "completion_tokens": 7197, "reasoning_chars": 19766, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/104", "ok": true, "error": null, "completion_tokens": 433, "reasoning_chars": 1116, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/105", "ok": true, "error": null, "completion_tokens": 771, "reasoning_chars": 1465, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/106", "ok": true, "error": null, "completion_tokens": 777, "reasoning_chars": 2294, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/107", "ok": true, "error": null, "completion_tokens": 592, "reasoning_chars": 993, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/108", "ok": true, "error": null, "completion_tokens": 1559, "reasoning_chars": 4324, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/109", "ok": true, "error": null, "completion_tokens": 970, "reasoning_chars": 2009, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/110", "ok": true, "error": null, "completion_tokens": 606, "reasoning_chars": 1298, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/111", "ok": true, "error": null, "completion_tokens": 688, "reasoning_chars": 1891, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/112", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/113", "ok": true, "error": null, "completion_tokens": 1434, "reasoning_chars": 4166, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/114", "ok": true, "error": null, "completion_tokens": 464, "reasoning_chars": 1156, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/115", "ok": true, "error": null, "completion_tokens": 531, "reasoning_chars": 677, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/116", "ok": false, "error": null, "completion_tokens": 24058, "reasoning_chars": 95239, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/117", "ok": true, "error": null, "completion_tokens": 488, "reasoning_chars": 1632, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/118", "ok": true, "error": null, "completion_tokens": 973, "reasoning_chars": 3337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/119", "ok": true, "error": null, "completion_tokens": 548, "reasoning_chars": 1110, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/120", "ok": true, "error": null, "completion_tokens": 616, "reasoning_chars": 1284, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/121", "ok": true, "error": null, "completion_tokens": 340, "reasoning_chars": 665, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/122", "ok": true, "error": null, "completion_tokens": 538, "reasoning_chars": 1400, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/123", "ok": true, "error": null, "completion_tokens": 896, "reasoning_chars": 2025, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/124", "ok": true, "error": null, "completion_tokens": 19279, "reasoning_chars": 66334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/125", "ok": true, "error": null, "completion_tokens": 962, "reasoning_chars": 2852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/126", "ok": true, "error": null, "completion_tokens": 715, "reasoning_chars": 2424, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/127", "ok": true, "error": null, "completion_tokens": 1401, "reasoning_chars": 3663, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/128", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 848, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/129", "ok": true, "error": null, "completion_tokens": 2746, "reasoning_chars": 9186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/130", "ok": true, "error": null, "completion_tokens": 3807, "reasoning_chars": 11044, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/131", "ok": true, "error": null, "completion_tokens": 385, "reasoning_chars": 942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/132", "ok": true, "error": null, "completion_tokens": 16446, "reasoning_chars": 67089, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/133", "ok": true, "error": null, "completion_tokens": 714, "reasoning_chars": 1866, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/134", "ok": false, "error": null, "completion_tokens": 11611, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/135", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 865, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/136", "ok": true, "error": null, "completion_tokens": 455, "reasoning_chars": 1026, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/137", "ok": true, "error": null, "completion_tokens": 823, "reasoning_chars": 2322, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/138", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 628, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/139", "ok": true, "error": null, "completion_tokens": 447, "reasoning_chars": 1025, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/140", "ok": true, "error": null, "completion_tokens": 1228, "reasoning_chars": 4060, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/141", "ok": true, "error": null, "completion_tokens": 1086, "reasoning_chars": 2696, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/142", "ok": false, "error": null, "completion_tokens": 10949, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/143", "ok": true, "error": null, "completion_tokens": 461, "reasoning_chars": 988, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/144", "ok": true, "error": null, "completion_tokens": 500, "reasoning_chars": 1138, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/145", "ok": false, "error": null, "completion_tokens": 16892, "reasoning_chars": 55672, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/146", "ok": true, "error": null, "completion_tokens": 1165, "reasoning_chars": 3094, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/147", "ok": true, "error": null, "completion_tokens": 1572, "reasoning_chars": 3597, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/148", "ok": true, "error": null, "completion_tokens": 660, "reasoning_chars": 2094, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/149", "ok": true, "error": null, "completion_tokens": 659, "reasoning_chars": 1765, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/150", "ok": false, "error": null, "completion_tokens": 382, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/151", "ok": true, "error": null, "completion_tokens": 1427, "reasoning_chars": 4627, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/152", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 490, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/153", "ok": true, "error": null, "completion_tokens": 1026, "reasoning_chars": 2533, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/154", "ok": true, "error": null, "completion_tokens": 619, "reasoning_chars": 2011, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/155", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 548, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/156", "ok": true, "error": null, "completion_tokens": 483, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/157", "ok": true, "error": null, "completion_tokens": 24480, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/158", "ok": true, "error": null, "completion_tokens": 782, "reasoning_chars": 2477, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/159", "ok": true, "error": null, "completion_tokens": 463, "reasoning_chars": 595, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/160", "ok": true, "error": null, "completion_tokens": 4242, "reasoning_chars": 14543, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/161", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 786, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/162", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 454, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/163", "ok": true, "error": null, "completion_tokens": 957, "reasoning_chars": 2384, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ifeval": { "task": "ifeval", "n": 100, "hits": 69, "accuracy_pct": 69.0, "accuracy_ex_trunc_pct": 69.0, "truncated": 0, "completion_tokens_p50": 372, "completion_tokens_p90": 2665, "completion_tokens_max": 15540, "reasoning_chars_p50": 34, "reasoning_chars_p90": 6615, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 1437.0, "gen_tokens": 114516, "tok_per_s": 79.7, "instruction_level_pct": 78.67, "instructions_checked": 150, "details": [ { "id": "ifeval-153", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-258", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-368", "ok": false, "error": null, "completion_tokens": 1236, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-85", "ok": false, "error": null, "completion_tokens": 299, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-246", "ok": true, "error": null, "completion_tokens": 1633, "reasoning_chars": 4566, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-23", "ok": true, "error": null, "completion_tokens": 2366, "reasoning_chars": 7469, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-372", "ok": true, "error": null, "completion_tokens": 2501, "reasoning_chars": 8288, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-387", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 1018, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-138", "ok": true, "error": null, "completion_tokens": 390, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-56", "ok": true, "error": null, "completion_tokens": 718, "reasoning_chars": 2572, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-277", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-407", "ok": false, "error": null, "completion_tokens": 992, "reasoning_chars": 2080, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-171", "ok": true, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-96", "ok": true, "error": null, "completion_tokens": 15540, "reasoning_chars": 20018, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-134", "ok": false, "error": null, "completion_tokens": 1624, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-218", "ok": true, "error": null, "completion_tokens": 828, "reasoning_chars": 1098, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-330", "ok": false, "error": null, "completion_tokens": 731, "reasoning_chars": 2372, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-6", "ok": true, "error": null, "completion_tokens": 270, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-143", "ok": true, "error": null, "completion_tokens": 1182, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-121", "ok": true, "error": null, "completion_tokens": 178, "reasoning_chars": 349, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-219", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-524", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-206", "ok": false, "error": null, "completion_tokens": 103, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-144", "ok": true, "error": null, "completion_tokens": 848, "reasoning_chars": 2415, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-447", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-492", "ok": false, "error": null, "completion_tokens": 3170, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-485", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-220", "ok": true, "error": null, "completion_tokens": 1032, "reasoning_chars": 3359, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-412", "ok": false, "error": null, "completion_tokens": 597, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-342", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-222", "ok": true, "error": null, "completion_tokens": 6629, "reasoning_chars": 18613, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-131", "ok": false, "error": null, "completion_tokens": 1436, "reasoning_chars": 5021, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-210", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 592, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-137", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-464", "ok": true, "error": null, "completion_tokens": 764, "reasoning_chars": 2764, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-503", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 458, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-446", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-2", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-525", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 1429, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-27", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-136", "ok": true, "error": null, "completion_tokens": 2529, "reasoning_chars": 7903, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-286", "ok": false, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-449", "ok": true, "error": null, "completion_tokens": 974, "reasoning_chars": 3672, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-55", "ok": false, "error": null, "completion_tokens": 132, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-93", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-399", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-19", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 700, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-229", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 241, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-181", "ok": true, "error": null, "completion_tokens": 772, "reasoning_chars": 2465, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-362", "ok": true, "error": null, "completion_tokens": 372, "reasoning_chars": 520, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-289", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-356", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-115", "ok": true, "error": null, "completion_tokens": 76, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-404", "ok": false, "error": null, "completion_tokens": 2605, "reasoning_chars": 7089, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-243", "ok": false, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-90", "ok": false, "error": null, "completion_tokens": 341, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-54", "ok": true, "error": null, "completion_tokens": 1698, "reasoning_chars": 4748, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-276", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-308", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-64", "ok": false, "error": null, "completion_tokens": 1346, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-260", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-337", "ok": true, "error": null, "completion_tokens": 741, "reasoning_chars": 2319, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-510", "ok": true, "error": null, "completion_tokens": 1458, "reasoning_chars": 3964, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-119", "ok": true, "error": null, "completion_tokens": 2549, "reasoning_chars": 8731, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-5", "ok": false, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-468", "ok": true, "error": null, "completion_tokens": 7722, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-418", "ok": false, "error": null, "completion_tokens": 241, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-476", "ok": false, "error": null, "completion_tokens": 5052, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-49", "ok": true, "error": null, "completion_tokens": 4020, "reasoning_chars": 8882, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-336", "ok": true, "error": null, "completion_tokens": 5590, "reasoning_chars": 20491, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-422", "ok": true, "error": null, "completion_tokens": 337, "reasoning_chars": 34, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-417", "ok": true, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-347", "ok": true, "error": null, "completion_tokens": 712, "reasoning_chars": 1870, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-101", "ok": true, "error": null, "completion_tokens": 1803, "reasoning_chars": 4663, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-209", "ok": true, "error": null, "completion_tokens": 1072, "reasoning_chars": 3222, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-341", "ok": true, "error": null, "completion_tokens": 1063, "reasoning_chars": 3179, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-533", "ok": true, "error": null, "completion_tokens": 278, "reasoning_chars": 723, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-158", "ok": false, "error": null, "completion_tokens": 552, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-321", "ok": true, "error": null, "completion_tokens": 2218, "reasoning_chars": 8793, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-311", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-50", "ok": false, "error": null, "completion_tokens": 63, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-518", "ok": true, "error": null, "completion_tokens": 779, "reasoning_chars": 1571, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-374", "ok": true, "error": null, "completion_tokens": 298, "reasoning_chars": 835, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-317", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 820, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-512", "ok": true, "error": null, "completion_tokens": 718, "reasoning_chars": 1721, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-102", "ok": false, "error": null, "completion_tokens": 2745, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-72", "ok": true, "error": null, "completion_tokens": 1284, "reasoning_chars": 5177, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-33", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-263", "ok": false, "error": null, "completion_tokens": 2665, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-501", "ok": false, "error": null, "completion_tokens": 5650, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-4", "ok": true, "error": null, "completion_tokens": 163, "reasoning_chars": 618, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-444", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-228", "ok": true, "error": null, "completion_tokens": 459, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-299", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-278", "ok": true, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-154", "ok": true, "error": null, "completion_tokens": 589, "reasoning_chars": 1997, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-301", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 1138, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-161", "ok": true, "error": null, "completion_tokens": 2724, "reasoning_chars": 6615, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-394", "ok": false, "error": null, "completion_tokens": 144, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-159", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 1055, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false } ] } } }, "original-gguf": { "started": "2026-10-10 15:40:38", "finished": "2026-10-10 18:50:07", "seed": 20260816, "thinking": true, "tasks": { "mmlu": { "task": "mmlu", "n": 150, "hits": 102, "accuracy_pct": 68.0, "accuracy_ex_trunc_pct": 68.0, "truncated": 0, "completion_tokens_p50": 53, "completion_tokens_p90": 672, "completion_tokens_max": 10397, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1508, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 7859.5, "gen_tokens": 52980, "tok_per_s": 6.7, "details": [ { "id": "mmlu-12325", "ok": true, "error": null, "completion_tokens": 1041, "reasoning_chars": 2458, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-154", "ok": false, "error": null, "completion_tokens": 196, "reasoning_chars": 515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12844", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1153", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11089", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12782", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5422", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 713, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12363", "ok": true, "error": null, "completion_tokens": 2776, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12556", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13163", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-270", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6943", "ok": true, "error": null, "completion_tokens": 472, "reasoning_chars": 2067, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8330", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 796, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9276", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7584", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 366, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12819", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6163", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 224, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4099", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 777, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10389", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 325, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1539", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12013", "ok": true, "error": null, "completion_tokens": 798, "reasoning_chars": 2459, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7814", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1106", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1616", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 560, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-555", "ok": true, "error": null, "completion_tokens": 395, "reasoning_chars": 1078, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3698", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-618", "ok": false, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9842", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 508, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4229", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 121, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1022", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 977, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6421", "ok": false, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1997", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 756, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8941", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-565", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11364", "ok": false, "error": null, "completion_tokens": 10397, "reasoning_chars": 39675, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5959", "ok": true, "error": null, "completion_tokens": 433, "reasoning_chars": 1355, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13066", "ok": false, "error": null, "completion_tokens": 398, "reasoning_chars": 1528, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3122", "ok": true, "error": null, "completion_tokens": 509, "reasoning_chars": 910, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3722", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 1508, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13839", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12570", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9959", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 145, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10791", "ok": false, "error": null, "completion_tokens": 1049, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12268", "ok": true, "error": null, "completion_tokens": 888, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9051", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12887", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2667", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-3906", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-10304", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12927", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6337", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 382, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3907", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12888", "ok": false, "error": null, "completion_tokens": 586, "reasoning_chars": 2247, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10629", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 462, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5801", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 1160, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2936", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7389", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9532", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-8705", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 286, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11776", "ok": true, "error": null, "completion_tokens": 1763, "reasoning_chars": 5868, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6836", "ok": false, "error": null, "completion_tokens": 3109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7856", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9697", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-10825", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11178", "ok": true, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11420", "ok": true, "error": null, "completion_tokens": 698, "reasoning_chars": 2098, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4658", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 893, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5403", "ok": false, "error": null, "completion_tokens": 672, "reasoning_chars": 2364, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6160", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6872", "ok": false, "error": null, "completion_tokens": 7933, "reasoning_chars": 32241, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9738", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 172, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3076", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 1296, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13496", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11289", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 598, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-222", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 145, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12180", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1143", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13086", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7038", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13587", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12349", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 720, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11747", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 1084, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9848", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 730, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3438", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 796, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-89", "ok": true, "error": null, "completion_tokens": 199, "reasoning_chars": 808, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4253", "ok": true, "error": null, "completion_tokens": 53, "reasoning_chars": 87, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5460", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2494", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11149", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-718", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5330", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-10716", "ok": true, "error": null, "completion_tokens": 1245, "reasoning_chars": 3862, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5636", "ok": true, "error": null, "completion_tokens": 691, "reasoning_chars": 1268, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11832", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-95", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 998, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13960", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 237, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2072", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 17, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12030", "ok": true, "error": null, "completion_tokens": 1477, "reasoning_chars": 5394, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2931", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 1015, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2441", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13666", "ok": true, "error": null, "completion_tokens": 344, "reasoning_chars": 1383, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6575", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 751, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11625", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13283", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-4262", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12931", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12150", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-3678", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-4593", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 1136, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1563", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13396", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-8049", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 676, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8396", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 902, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8493", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13514", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 279, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3000", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 111, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1635", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 497, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1391", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-627", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1573", "ok": true, "error": null, "completion_tokens": 414, "reasoning_chars": 853, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6731", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11665", "ok": true, "error": null, "completion_tokens": 815, "reasoning_chars": 2355, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2914", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6875", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 1344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2184", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 735, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12801", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6717", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11", "ok": true, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12464", "ok": false, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2787", "ok": false, "error": null, "completion_tokens": 53, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11549", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-842", "ok": true, "error": null, "completion_tokens": 514, "reasoning_chars": 1791, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13868", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9647", "ok": true, "error": null, "completion_tokens": 370, "reasoning_chars": 1430, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9502", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12441", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 1358, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4392", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5036", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 737, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12606", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1400", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 335, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13979", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 712, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13634", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 1092, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11226", "ok": true, "error": null, "completion_tokens": 1435, "reasoning_chars": 4633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-625", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-257", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12853", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 583, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-451", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2074", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 42, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9117", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 493, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10748", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "cmmlu": { "task": "cmmlu", "n": 150, "hits": 92, "accuracy_pct": 61.33, "accuracy_ex_trunc_pct": 61.33, "truncated": 0, "completion_tokens_p50": 14, "completion_tokens_p90": 292, "completion_tokens_max": 1400, "reasoning_chars_p50": 0, "reasoning_chars_p90": 515, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 7772.0, "gen_tokens": 16569, "tok_per_s": 2.1, "details": [ { "id": "cmmlu-chinese_teacher_qualification-63", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 303, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-54", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 254, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-91", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-260", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-32", "ok": true, "error": null, "completion_tokens": 48, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-29", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-200", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-98", "ok": true, "error": null, "completion_tokens": 53, "reasoning_chars": 69, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-44", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 208, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-118", "ok": true, "error": null, "completion_tokens": 236, "reasoning_chars": 774, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-116", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-43", "ok": true, "error": null, "completion_tokens": 364, "reasoning_chars": 515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-145", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-247", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 339, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-224", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 299, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-26", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-105", "ok": false, "error": null, "completion_tokens": 461, "reasoning_chars": 690, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-15", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-62", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-nutrition-74", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-181", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 159, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-127", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_geography-107", "ok": false, "error": null, "completion_tokens": 223, "reasoning_chars": 288, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-18", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-82", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_food_culture-38", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 150, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-198", "ok": true, "error": null, "completion_tokens": 815, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-96", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 89, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-73", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-20", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-95", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-88", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-192", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-66", "ok": true, "error": null, "completion_tokens": 248, "reasoning_chars": 184, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-100", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-136", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-16", "ok": false, "error": null, "completion_tokens": 298, "reasoning_chars": 643, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-90", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 1287, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-41", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-345", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-103", "ok": false, "error": null, "completion_tokens": 135, "reasoning_chars": 222, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-93", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-88", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-206", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-105", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-90", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-190", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-international_law-39", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-155", "ok": true, "error": null, "completion_tokens": 1369, "reasoning_chars": 4689, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-24", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-6", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-31", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-243", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-63", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-135", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 63, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-89", "ok": false, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-91", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-120", "ok": false, "error": null, "completion_tokens": 375, "reasoning_chars": 769, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-62", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-106", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 150, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-food_science-106", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-24", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-157", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-41", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-115", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-journalism-29", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 110, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-67", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 91, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-161", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-120", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-education-117", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-169", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 217, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-129", "ok": true, "error": null, "completion_tokens": 175, "reasoning_chars": 271, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-19", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-8", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-72", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 571, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-139", "ok": false, "error": null, "completion_tokens": 238, "reasoning_chars": 372, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_security-22", "ok": false, "error": null, "completion_tokens": 82, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-7", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-108", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-31", "ok": true, "error": null, "completion_tokens": 27, "reasoning_chars": 29, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-131", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-35", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_actuarial_science-38", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-165", "ok": true, "error": null, "completion_tokens": 64, "reasoning_chars": 88, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-101", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-107", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 128, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-8", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medical_statistics-86", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-security_study-30", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 63, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-118", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_law-24", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-46", "ok": true, "error": null, "completion_tokens": 689, "reasoning_chars": 2272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-anatomy-21", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-145", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-91", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 204, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-109", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 9, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-44", "ok": false, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-74", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-63", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-146", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-109", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 786, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-construction_project_management-16", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-113", "ok": false, "error": null, "completion_tokens": 174, "reasoning_chars": 307, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-102", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-207", "ok": false, "error": null, "completion_tokens": 194, "reasoning_chars": 262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-19", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-164", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 305, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-338", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 55, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-7", "ok": true, "error": null, "completion_tokens": 599, "reasoning_chars": 1797, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-92", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-47", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 1131, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-140", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-100", "ok": false, "error": null, "completion_tokens": 89, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-37", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-191", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-34", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-88", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-77", "ok": false, "error": null, "completion_tokens": 404, "reasoning_chars": 1212, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-306", "ok": true, "error": null, "completion_tokens": 892, "reasoning_chars": 2543, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-40", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 68, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_chemistry-7", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-152", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 236, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-1", "ok": true, "error": null, "completion_tokens": 119, "reasoning_chars": 170, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-94", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 135, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-109", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 356, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-164", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-89", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-17", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-2", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 175, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-51", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-141", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 417, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-196", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-97", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-202", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-195", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-49", "ok": true, "error": null, "completion_tokens": 1400, "reasoning_chars": 2633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-94", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-8", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-98", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 178, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-28", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-74", "ok": false, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-121", "ok": true, "error": null, "completion_tokens": 437, "reasoning_chars": 813, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-120", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-105", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 123, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-35", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 276, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-51", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-49", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-130", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-140", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "mmlu_pro": { "task": "mmlu_pro", "n": 150, "hits": 96, "accuracy_pct": 64.0, "accuracy_ex_trunc_pct": 64.43, "truncated": 1, "completion_tokens_p50": 61, "completion_tokens_p90": 1300, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 2971, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 7964.7, "gen_tokens": 121349, "tok_per_s": 15.2, "details": [ { "id": "mmlupro-5831", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2299", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9280", "ok": false, "error": null, "completion_tokens": 8540, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5486", "ok": false, "error": null, "completion_tokens": 500, "reasoning_chars": 2138, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5761", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6589", "ok": true, "error": null, "completion_tokens": 876, "reasoning_chars": 2114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5389", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5131", "ok": false, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1677", "ok": true, "error": null, "completion_tokens": 1373, "reasoning_chars": 4868, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4388", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3285", "ok": true, "error": null, "completion_tokens": 578, "reasoning_chars": 1608, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4118", "ok": true, "error": null, "completion_tokens": 739, "reasoning_chars": 1816, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5344", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11696", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-858", "ok": true, "error": null, "completion_tokens": 114, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-214", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6609", "ok": true, "error": null, "completion_tokens": 991, "reasoning_chars": 2835, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2083", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9752", "ok": false, "error": null, "completion_tokens": 671, "reasoning_chars": 1613, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7213", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5234", "ok": false, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8570", "ok": false, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5612", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2071", "ok": true, "error": null, "completion_tokens": 597, "reasoning_chars": 2000, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9842", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 100, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4665", "ok": true, "error": null, "completion_tokens": 1026, "reasoning_chars": 2282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5564", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6475", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 738, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2153", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5812", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 487, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3407", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2879", "ok": false, "error": null, "completion_tokens": 910, "reasoning_chars": 1851, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8266", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 286, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1916", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3486", "ok": false, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11275", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6040", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8565", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 256, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6848", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8926", "ok": true, "error": null, "completion_tokens": 277, "reasoning_chars": 344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8458", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 88, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7854", "ok": true, "error": null, "completion_tokens": 375, "reasoning_chars": 964, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3962", "ok": true, "error": null, "completion_tokens": 1980, "reasoning_chars": 1928, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6470", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3813", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 763, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8026", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11030", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8151", "ok": true, "error": null, "completion_tokens": 62, "reasoning_chars": 81, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5286", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3888", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 119, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11676", "ok": true, "error": null, "completion_tokens": 1768, "reasoning_chars": 3608, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8762", "ok": false, "error": null, "completion_tokens": 973, "reasoning_chars": 1949, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-12160", "ok": true, "error": null, "completion_tokens": 2014, "reasoning_chars": 4691, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4022", "ok": true, "error": null, "completion_tokens": 1299, "reasoning_chars": 3196, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10092", "ok": false, "error": null, "completion_tokens": 6003, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7364", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 1442, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6001", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10873", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9777", "ok": false, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8723", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4595", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 872, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8423", "ok": true, "error": null, "completion_tokens": 7326, "reasoning_chars": 10052, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11051", "ok": false, "error": null, "completion_tokens": 225, "reasoning_chars": 789, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-836", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11297", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 82902, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlupro-3950", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1562", "ok": false, "error": null, "completion_tokens": 779, "reasoning_chars": 2746, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2041", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3473", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 1528, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11253", "ok": true, "error": null, "completion_tokens": 828, "reasoning_chars": 2971, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3858", "ok": true, "error": null, "completion_tokens": 479, "reasoning_chars": 847, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2231", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2190", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6930", "ok": true, "error": null, "completion_tokens": 471, "reasoning_chars": 1804, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6558", "ok": true, "error": null, "completion_tokens": 824, "reasoning_chars": 2267, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5371", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1766", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6045", "ok": true, "error": null, "completion_tokens": 1006, "reasoning_chars": 3100, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2298", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6400", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6499", "ok": true, "error": null, "completion_tokens": 98, "reasoning_chars": 378, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7936", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 1058, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8566", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11633", "ok": false, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9843", "ok": false, "error": null, "completion_tokens": 7898, "reasoning_chars": 19417, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11758", "ok": true, "error": null, "completion_tokens": 9174, "reasoning_chars": 16853, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2843", "ok": true, "error": null, "completion_tokens": 1313, "reasoning_chars": 3768, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7483", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11379", "ok": false, "error": null, "completion_tokens": 51, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4866", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-818", "ok": false, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2785", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8119", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 1293, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2214", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5774", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6049", "ok": false, "error": null, "completion_tokens": 415, "reasoning_chars": 1169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7219", "ok": true, "error": null, "completion_tokens": 492, "reasoning_chars": 1788, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1660", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4767", "ok": true, "error": null, "completion_tokens": 1300, "reasoning_chars": 5100, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5964", "ok": false, "error": null, "completion_tokens": 398, "reasoning_chars": 944, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2743", "ok": true, "error": null, "completion_tokens": 753, "reasoning_chars": 2620, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4062", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 704, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7368", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-827", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11877", "ok": false, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7557", "ok": true, "error": null, "completion_tokens": 886, "reasoning_chars": 3082, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9998", "ok": true, "error": null, "completion_tokens": 218, "reasoning_chars": 334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10098", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 880, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1744", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9210", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 328, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11248", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 744, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11187", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2799", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4479", "ok": true, "error": null, "completion_tokens": 1518, "reasoning_chars": 3302, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10223", "ok": true, "error": null, "completion_tokens": 540, "reasoning_chars": 1166, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5698", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8291", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 416, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3516", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5466", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4851", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-733", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6995", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2573", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8481", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5938", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10708", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8476", "ok": false, "error": null, "completion_tokens": 56, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7990", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 794, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2409", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11551", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-674", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1978", "ok": true, "error": null, "completion_tokens": 769, "reasoning_chars": 2348, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10420", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3445", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9488", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 571, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4993", "ok": true, "error": null, "completion_tokens": 501, "reasoning_chars": 1746, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4316", "ok": true, "error": null, "completion_tokens": 29, "reasoning_chars": 29, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4730", "ok": true, "error": null, "completion_tokens": 411, "reasoning_chars": 1328, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11570", "ok": false, "error": null, "completion_tokens": 85, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2149", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8322", "ok": true, "error": null, "completion_tokens": 1187, "reasoning_chars": 2275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10993", "ok": false, "error": null, "completion_tokens": 794, "reasoning_chars": 2418, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4588", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3258", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8629", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-637", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 89, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4257", "ok": true, "error": null, "completion_tokens": 427, "reasoning_chars": 931, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9719", "ok": true, "error": null, "completion_tokens": 1810, "reasoning_chars": 4454, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11714", "ok": false, "error": null, "completion_tokens": 6315, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9678", "ok": true, "error": null, "completion_tokens": 1470, "reasoning_chars": 5351, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ceval": { "task": "ceval", "n": 150, "hits": 90, "accuracy_pct": 60.0, "accuracy_ex_trunc_pct": 60.0, "truncated": 0, "completion_tokens_p50": 12, "completion_tokens_p90": 448, "completion_tokens_max": 31502, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1074, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 3097.4, "gen_tokens": 54577, "tok_per_s": 17.6, "details": [ { "id": "ceval-high_school_chinese-1", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-10", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-41", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-3", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-21", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 563, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-4", "ok": true, "error": null, "completion_tokens": 1904, "reasoning_chars": 3988, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-21", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-2", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 124, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-6", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-12", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-19", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chemistry-9", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-15", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-22", "ok": false, "error": null, "completion_tokens": 101, "reasoning_chars": 184, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-16", "ok": true, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-11", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-18", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 161, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-24", "ok": true, "error": null, "completion_tokens": 531, "reasoning_chars": 1605, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-29", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-2", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-11", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-logic-18", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-0", "ok": false, "error": null, "completion_tokens": 41, "reasoning_chars": 52, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-14", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-27", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-19", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 149, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-21", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-33", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 348, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-20", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-10", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-19", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-7", "ok": true, "error": null, "completion_tokens": 42, "reasoning_chars": 58, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-41", "ok": true, "error": null, "completion_tokens": 617, "reasoning_chars": 1972, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-44", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 79, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-9", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-28", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-39", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-1", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-24", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-5", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-computer_architecture-15", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-39", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 1756, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-23", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-7", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-17", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 1171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-6", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-15", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 119, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-4", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-logic-5", "ok": true, "error": null, "completion_tokens": 1104, "reasoning_chars": 3434, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-50", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 257, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_politics-11", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-32", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-logic-16", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-9", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-15", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-2", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-7", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 381, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-11", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 230, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-operating_system-18", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 135, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-4", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-17", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-1", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 301, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-32", "ok": true, "error": null, "completion_tokens": 86, "reasoning_chars": 203, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-33", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-25", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-40", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 90, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-7", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-5", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 1889, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-12", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-4", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-38", "ok": false, "error": null, "completion_tokens": 31502, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-22", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-15", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-45", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 101, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-6", "ok": false, "error": null, "completion_tokens": 40, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-20", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-2", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-17", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-16", "ok": false, "error": null, "completion_tokens": 620, "reasoning_chars": 2508, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-7", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_biology-16", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-10", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-32", "ok": false, "error": null, "completion_tokens": 2280, "reasoning_chars": 7445, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-0", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-12", "ok": false, "error": null, "completion_tokens": 1735, "reasoning_chars": 2714, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-14", "ok": true, "error": null, "completion_tokens": 672, "reasoning_chars": 2546, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-26", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-3", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-3", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 43, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-13", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-5", "ok": true, "error": null, "completion_tokens": 448, "reasoning_chars": 369, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-27", "ok": true, "error": null, "completion_tokens": 2603, "reasoning_chars": 3916, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-10", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 41, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-16", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-49", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-13", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-5", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-2", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 93, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-27", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_biology-8", "ok": true, "error": null, "completion_tokens": 95, "reasoning_chars": 126, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-14", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-14", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-31", "ok": true, "error": null, "completion_tokens": 459, "reasoning_chars": 836, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-13", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-12", "ok": true, "error": null, "completion_tokens": 589, "reasoning_chars": 1582, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-44", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-3", "ok": true, "error": null, "completion_tokens": 1213, "reasoning_chars": 3337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-13", "ok": true, "error": null, "completion_tokens": 1089, "reasoning_chars": 1074, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-6", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_politics-19", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-environmental_impact_assessment_engineer-27", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-law-10", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-3", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 212, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-6", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chinese-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-43", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-42", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-discrete_mathematics-1", "ok": true, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-11", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-16", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 101, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-2", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 139, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-1", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-16", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-11", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 218, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-5", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 281, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-40", "ok": true, "error": null, "completion_tokens": 486, "reasoning_chars": 1428, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-6", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 211, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-10", "ok": false, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-10", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-17", "ok": false, "error": null, "completion_tokens": 113, "reasoning_chars": 219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-11", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-16", "ok": true, "error": null, "completion_tokens": 65, "reasoning_chars": 89, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-1", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-0", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-37", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-9", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 96, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_history-16", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-5", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-18", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-30", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-42", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-5", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-4", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "arc": { "task": "arc", "n": 150, "hits": 135, "accuracy_pct": 90.0, "accuracy_ex_trunc_pct": 90.0, "truncated": 0, "completion_tokens_p50": 225, "completion_tokens_p90": 443, "completion_tokens_max": 1572, "reasoning_chars_p50": 777, "reasoning_chars_p90": 1302, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 7769.6, "gen_tokens": 37617, "tok_per_s": 4.8, "details": [ { "id": "arc-Mercury_415084", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 48, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7249708", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 358, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_4_24", "ok": true, "error": null, "completion_tokens": 242, "reasoning_chars": 905, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7056823", "ok": true, "error": null, "completion_tokens": 268, "reasoning_chars": 755, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415534", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 525, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405827", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263183", "ok": true, "error": null, "completion_tokens": 261, "reasoning_chars": 712, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106960", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 256, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_7", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 233, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7182245", "ok": true, "error": null, "completion_tokens": 623, "reasoning_chars": 1874, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416177", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2011_5_8", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 1041, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_8_36", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 90, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401699", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 163, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107240", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 207, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_4_26", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 728, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402638", "ok": true, "error": null, "completion_tokens": 502, "reasoning_chars": 1702, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026723", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 273, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415422", "ok": true, "error": null, "completion_tokens": 275, "reasoning_chars": 961, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2000_8_2", "ok": true, "error": null, "completion_tokens": 403, "reasoning_chars": 1163, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2006_9_4", "ok": true, "error": null, "completion_tokens": 562, "reasoning_chars": 1660, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401272", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 391, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416169", "ok": true, "error": null, "completion_tokens": 357, "reasoning_chars": 799, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7116235", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 1325, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7109463", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7032428", "ok": false, "error": null, "completion_tokens": 841, "reasoning_chars": 2498, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2008_8_11", "ok": true, "error": null, "completion_tokens": 349, "reasoning_chars": 1074, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_410334", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 1002, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_36", "ok": true, "error": null, "completion_tokens": 90, "reasoning_chars": 279, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415719", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 448, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_17", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 807, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402256", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 547, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_400443", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7133245", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 781, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413458", "ok": true, "error": null, "completion_tokens": 48, "reasoning_chars": 182, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401372", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 714, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401116", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 359, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__5_10312", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 911, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2007_4_10280", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 437, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416683", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 1500, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7190120", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 867, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401295", "ok": true, "error": null, "completion_tokens": 362, "reasoning_chars": 1439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_4_28", "ok": true, "error": null, "completion_tokens": 85, "reasoning_chars": 347, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7223493", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018410", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 339, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1998_8_24", "ok": true, "error": null, "completion_tokens": 762, "reasoning_chars": 2272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_31", "ok": true, "error": null, "completion_tokens": 316, "reasoning_chars": 822, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405501", "ok": true, "error": null, "completion_tokens": 297, "reasoning_chars": 1012, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408030", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 842, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401684", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 690, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041668", "ok": true, "error": null, "completion_tokens": 403, "reasoning_chars": 1292, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7205870", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 963, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7210140", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 1234, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405873", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 1127, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400052", "ok": true, "error": null, "completion_tokens": 211, "reasoning_chars": 606, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018270", "ok": true, "error": null, "completion_tokens": 455, "reasoning_chars": 1187, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406550", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 810, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7150728", "ok": true, "error": null, "completion_tokens": 514, "reasoning_chars": 1289, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400839", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 710, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_30", "ok": false, "error": null, "completion_tokens": 179, "reasoning_chars": 864, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400383", "ok": true, "error": null, "completion_tokens": 479, "reasoning_chars": 1302, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7236740", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-CSZ_2005_5_CSZ10021", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 883, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7093958", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 816, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026355", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 230, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408324", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 332, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__7_10348", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402629", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 190, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400046", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 1007, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7180810", "ok": true, "error": null, "completion_tokens": 207, "reasoning_chars": 1025, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_180005", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 760, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2009_8_14", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2007_5_16", "ok": true, "error": null, "completion_tokens": 32, "reasoning_chars": 134, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_LBS10662", "ok": true, "error": null, "completion_tokens": 133, "reasoning_chars": 398, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_23", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 811, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2012_8_19", "ok": false, "error": null, "completion_tokens": 118, "reasoning_chars": 524, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041650", "ok": true, "error": null, "completion_tokens": 359, "reasoning_chars": 818, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416181", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 325, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_1995_8_J6", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 998, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2014_5_13", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 526, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106750", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 962, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_4_23", "ok": true, "error": null, "completion_tokens": 498, "reasoning_chars": 1511, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MEA_2013_5_12", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 195, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_8_42", "ok": false, "error": null, "completion_tokens": 177, "reasoning_chars": 427, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408367", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 368, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7250058", "ok": true, "error": null, "completion_tokens": 350, "reasoning_chars": 979, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_414041", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 674, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401290", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409295", "ok": true, "error": null, "completion_tokens": 310, "reasoning_chars": 1016, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7211680", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 1199, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_37", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 898, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413439", "ok": true, "error": null, "completion_tokens": 413, "reasoning_chars": 1211, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_32", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 777, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7216423", "ok": true, "error": null, "completion_tokens": 225, "reasoning_chars": 1106, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2015_7_9", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7230405", "ok": true, "error": null, "completion_tokens": 514, "reasoning_chars": 1477, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7025008", "ok": true, "error": null, "completion_tokens": 483, "reasoning_chars": 1319, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416536", "ok": true, "error": null, "completion_tokens": 261, "reasoning_chars": 760, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_2007_8_pg60", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_4_34", "ok": true, "error": null, "completion_tokens": 439, "reasoning_chars": 1287, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_406833", "ok": true, "error": null, "completion_tokens": 390, "reasoning_chars": 1014, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409647", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 1202, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_184363", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107170", "ok": true, "error": null, "completion_tokens": 286, "reasoning_chars": 722, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416104", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 167, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_10-v1", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 176, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7202843", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029803", "ok": true, "error": null, "completion_tokens": 360, "reasoning_chars": 1175, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7164623", "ok": true, "error": null, "completion_tokens": 271, "reasoning_chars": 896, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_26", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 222, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263060", "ok": true, "error": null, "completion_tokens": 390, "reasoning_chars": 1003, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268328", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 212, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402282", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 290, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029558", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 205, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_4_21", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 397, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7186358", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 1084, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2011_8_10434", "ok": true, "error": null, "completion_tokens": 420, "reasoning_chars": 1166, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2012_8_23651", "ok": true, "error": null, "completion_tokens": 246, "reasoning_chars": 624, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_407125", "ok": true, "error": null, "completion_tokens": 1572, "reasoning_chars": 5303, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268258", "ok": false, "error": null, "completion_tokens": 269, "reasoning_chars": 860, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7027405", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 594, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_8_7", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 1086, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_191625", "ok": false, "error": null, "completion_tokens": 718, "reasoning_chars": 3139, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_6", "ok": true, "error": null, "completion_tokens": 159, "reasoning_chars": 177, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7219678", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 1108, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401678", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 165, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401312", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_8_42", "ok": false, "error": null, "completion_tokens": 82, "reasoning_chars": 167, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7030468", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 1114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175385", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 1063, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175560", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_5", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 137, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_409009", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 966, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7128923", "ok": false, "error": null, "completion_tokens": 280, "reasoning_chars": 1310, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_8_9", "ok": false, "error": null, "completion_tokens": 170, "reasoning_chars": 589, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7042945", "ok": true, "error": null, "completion_tokens": 309, "reasoning_chars": 781, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_8_47", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 646, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7185343", "ok": true, "error": null, "completion_tokens": 536, "reasoning_chars": 1523, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7218400", "ok": true, "error": null, "completion_tokens": 340, "reasoning_chars": 922, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7221988", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 534, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405950", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 514, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7222530", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 971, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7102323", "ok": true, "error": null, "completion_tokens": 131, "reasoning_chars": 361, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_5_39", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 580, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_8_5", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 424, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400661", "ok": true, "error": null, "completion_tokens": 380, "reasoning_chars": 999, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7081603", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 890, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2008_3_25", "ok": true, "error": null, "completion_tokens": 158, "reasoning_chars": 420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413628", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 392, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406427", "ok": true, "error": null, "completion_tokens": 433, "reasoning_chars": 1029, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "truthfulqa": { "task": "truthfulqa", "n": 150, "hits": 91, "accuracy_pct": 60.67, "accuracy_ex_trunc_pct": 60.67, "truncated": 0, "completion_tokens_p50": 6, "completion_tokens_p90": 119, "completion_tokens_max": 658, "reasoning_chars_p50": 0, "reasoning_chars_p90": 284, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 4535.0, "gen_tokens": 6431, "tok_per_s": 1.4, "details": [ { "id": "tqa-210", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-76", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-461", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-218", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-254", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-635", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-193", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-148", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-400", "ok": false, "error": null, "completion_tokens": 180, "reasoning_chars": 661, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-766", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-240", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-50", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-83", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-381", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-325", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-655", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-721", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-391", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-401", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-625", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-80", "ok": false, "error": null, "completion_tokens": 322, "reasoning_chars": 852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-439", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-66", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-155", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-516", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-522", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-47", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-343", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-126", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-77", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-613", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-590", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-674", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-438", "ok": false, "error": null, "completion_tokens": 251, "reasoning_chars": 709, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-503", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-639", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-111", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-289", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-705", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-373", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-363", "ok": true, "error": null, "completion_tokens": 529, "reasoning_chars": 1831, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-792", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-113", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-74", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-634", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 682, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-804", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-626", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-676", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-174", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-455", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-725", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-296", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-535", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-172", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-23", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-43", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-38", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-97", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-657", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-814", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-346", "ok": true, "error": null, "completion_tokens": 264, "reasoning_chars": 924, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-365", "ok": true, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-384", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-677", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-486", "ok": false, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-55", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-680", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-510", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-128", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 1300, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-805", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-394", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-435", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-285", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-687", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-588", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-100", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-399", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-266", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-693", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-532", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-558", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-319", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-459", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-675", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 1318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-129", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-331", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-12", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-164", "ok": false, "error": null, "completion_tokens": 582, "reasoning_chars": 1580, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-306", "ok": true, "error": null, "completion_tokens": 119, "reasoning_chars": 284, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-275", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-418", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-264", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-94", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-389", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-427", "ok": true, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-269", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-643", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-7", "ok": true, "error": null, "completion_tokens": 658, "reasoning_chars": 1983, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-419", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-616", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-699", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-473", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-523", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-214", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 489, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-376", "ok": true, "error": null, "completion_tokens": 56, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-123", "ok": true, "error": null, "completion_tokens": 203, "reasoning_chars": 785, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-200", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-493", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-474", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-387", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-746", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-790", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-181", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-413", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-349", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-434", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-601", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-489", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-286", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-488", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-759", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-273", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-59", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-315", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-425", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-58", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-691", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-497", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-515", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-398", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-670", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-170", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-334", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-307", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-495", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 928, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-597", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-775", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-186", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 841, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-733", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-308", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-5", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-231", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 751, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-356", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-810", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-270", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-13", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-114", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-4", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-417", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-750", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "gsm8k": { "task": "gsm8k", "n": 100, "hits": 37, "accuracy_pct": 37.0, "accuracy_ex_trunc_pct": 37.0, "truncated": 0, "completion_tokens_p50": 7, "completion_tokens_p90": 17, "completion_tokens_max": 2152, "reasoning_chars_p50": 0, "reasoning_chars_p90": 0, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 5886.8, "gen_tokens": 5333, "tok_per_s": 0.9, "details": [ { "id": "gsm8k-144", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-627", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-195", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-201", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-730", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-166", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1141", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-832", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-53", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1042", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-609", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-318", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-892", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-204", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1120", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1040", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-556", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1015", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-206", "ok": true, "error": null, "completion_tokens": 282, "reasoning_chars": 582, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1058", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1044", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-131", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1154", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-158", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1149", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1156", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-60", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-918", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1010", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1032", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1053", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-577", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-935", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-692", "ok": true, "error": null, "completion_tokens": 259, "reasoning_chars": 407, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-153", "ok": true, "error": null, "completion_tokens": 698, "reasoning_chars": 1693, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-40", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-613", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-875", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-189", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-800", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-169", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1169", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-211", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1183", "ok": true, "error": null, "completion_tokens": 2152, "reasoning_chars": 8585, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-193", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-230", "ok": false, "error": null, "completion_tokens": 71, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-787", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-83", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1318", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-733", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1205", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-88", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-509", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-311", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-874", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-287", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 449, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1055", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1246", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-975", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-280", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-541", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-974", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-827", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-125", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-453", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-942", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1199", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-884", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-855", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1270", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-349", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1231", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1106", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-803", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-922", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-152", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-920", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-207", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-297", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-786", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-101", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1301", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-27", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1005", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-659", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-340", "ok": true, "error": null, "completion_tokens": 876, "reasoning_chars": 3280, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-288", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-763", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-840", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-742", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-222", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-468", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-220", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-662", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-269", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1193", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-270", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-114", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-728", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-396", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "math500": { "task": "math500", "n": 100, "hits": 68, "accuracy_pct": 68.0, "accuracy_ex_trunc_pct": 68.0, "truncated": 0, "completion_tokens_p50": 364, "completion_tokens_p90": 3353, "completion_tokens_max": 23656, "reasoning_chars_p50": 625, "reasoning_chars_p90": 7342, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 10017.1, "gen_tokens": 121453, "tok_per_s": 12.1, "details": [ { "id": "math-test/algebra/1553.json", "ok": true, "error": null, "completion_tokens": 496, "reasoning_chars": 750, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/538.json", "ok": true, "error": null, "completion_tokens": 525, "reasoning_chars": 661, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/691.json", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1090.json", "ok": true, "error": null, "completion_tokens": 672, "reasoning_chars": 1440, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/116.json", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1255.json", "ok": true, "error": null, "completion_tokens": 624, "reasoning_chars": 1837, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1143.json", "ok": true, "error": null, "completion_tokens": 684, "reasoning_chars": 895, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1201.json", "ok": true, "error": null, "completion_tokens": 23656, "reasoning_chars": 53668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/964.json", "ok": true, "error": null, "completion_tokens": 3512, "reasoning_chars": 6056, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/662.json", "ok": false, "error": null, "completion_tokens": 4064, "reasoning_chars": 7342, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/153.json", "ok": false, "error": null, "completion_tokens": 222, "reasoning_chars": 346, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/276.json", "ok": true, "error": null, "completion_tokens": 418, "reasoning_chars": 611, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1840.json", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 410, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1072.json", "ok": false, "error": null, "completion_tokens": 436, "reasoning_chars": 618, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1907.json", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/567.json", "ok": true, "error": null, "completion_tokens": 364, "reasoning_chars": 533, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1922.json", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1779.json", "ok": true, "error": null, "completion_tokens": 4705, "reasoning_chars": 11788, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/951.json", "ok": true, "error": null, "completion_tokens": 302, "reasoning_chars": 444, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1197.json", "ok": false, "error": null, "completion_tokens": 9506, "reasoning_chars": 18657, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/853.json", "ok": true, "error": null, "completion_tokens": 853, "reasoning_chars": 1276, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/516.json", "ok": false, "error": null, "completion_tokens": 1255, "reasoning_chars": 3186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1114.json", "ok": false, "error": null, "completion_tokens": 183, "reasoning_chars": 374, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1065.json", "ok": true, "error": null, "completion_tokens": 462, "reasoning_chars": 598, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1232.json", "ok": false, "error": null, "completion_tokens": 291, "reasoning_chars": 305, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/769.json", "ok": true, "error": null, "completion_tokens": 558, "reasoning_chars": 566, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1791.json", "ok": false, "error": null, "completion_tokens": 1193, "reasoning_chars": 2837, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/246.json", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 293, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1166.json", "ok": true, "error": null, "completion_tokens": 4882, "reasoning_chars": 8653, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1410.json", "ok": false, "error": null, "completion_tokens": 394, "reasoning_chars": 606, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1686.json", "ok": true, "error": null, "completion_tokens": 160, "reasoning_chars": 334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/754.json", "ok": false, "error": null, "completion_tokens": 697, "reasoning_chars": 1422, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/477.json", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 1349, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2680.json", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 340, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/23957.json", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1865.json", "ok": true, "error": null, "completion_tokens": 604, "reasoning_chars": 1337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1014.json", "ok": true, "error": null, "completion_tokens": 338, "reasoning_chars": 715, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/431.json", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 271, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1425.json", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 237, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2584.json", "ok": false, "error": null, "completion_tokens": 195, "reasoning_chars": 260, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2470.json", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 50, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/547.json", "ok": true, "error": null, "completion_tokens": 2677, "reasoning_chars": 7832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1105.json", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 503, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/347.json", "ok": true, "error": null, "completion_tokens": 236, "reasoning_chars": 625, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1257.json", "ok": true, "error": null, "completion_tokens": 429, "reasoning_chars": 882, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1126.json", "ok": true, "error": null, "completion_tokens": 529, "reasoning_chars": 747, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/239.json", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1961.json", "ok": false, "error": null, "completion_tokens": 306, "reasoning_chars": 817, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1388.json", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1078.json", "ok": true, "error": null, "completion_tokens": 725, "reasoning_chars": 1399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2157.json", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 494, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/339.json", "ok": true, "error": null, "completion_tokens": 1031, "reasoning_chars": 3239, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1761.json", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 260, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1960.json", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2086.json", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1462.json", "ok": true, "error": null, "completion_tokens": 6188, "reasoning_chars": 11376, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1060.json", "ok": true, "error": null, "completion_tokens": 798, "reasoning_chars": 981, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1055.json", "ok": true, "error": null, "completion_tokens": 2924, "reasoning_chars": 8058, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2517.json", "ok": false, "error": null, "completion_tokens": 486, "reasoning_chars": 933, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/996.json", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/893.json", "ok": true, "error": null, "completion_tokens": 465, "reasoning_chars": 498, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/51.json", "ok": false, "error": null, "completion_tokens": 330, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1291.json", "ok": true, "error": null, "completion_tokens": 438, "reasoning_chars": 670, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/694.json", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/711.json", "ok": true, "error": null, "completion_tokens": 5909, "reasoning_chars": 18936, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1655.json", "ok": false, "error": null, "completion_tokens": 260, "reasoning_chars": 419, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1936.json", "ok": true, "error": null, "completion_tokens": 144, "reasoning_chars": 234, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1572.json", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 690, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/509.json", "ok": false, "error": null, "completion_tokens": 476, "reasoning_chars": 763, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/864.json", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1405.json", "ok": true, "error": null, "completion_tokens": 1680, "reasoning_chars": 4006, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/192.json", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2102.json", "ok": true, "error": null, "completion_tokens": 347, "reasoning_chars": 519, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/843.json", "ok": true, "error": null, "completion_tokens": 418, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/187.json", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 433, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/612.json", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 1115, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/615.json", "ok": true, "error": null, "completion_tokens": 1085, "reasoning_chars": 3063, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1908.json", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/928.json", "ok": true, "error": null, "completion_tokens": 995, "reasoning_chars": 2230, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/505.json", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1930.json", "ok": false, "error": null, "completion_tokens": 3353, "reasoning_chars": 6577, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1934.json", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 204, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1139.json", "ok": true, "error": null, "completion_tokens": 9872, "reasoning_chars": 28089, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1000.json", "ok": true, "error": null, "completion_tokens": 2432, "reasoning_chars": 6596, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/183.json", "ok": true, "error": null, "completion_tokens": 486, "reasoning_chars": 775, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1275.json", "ok": true, "error": null, "completion_tokens": 571, "reasoning_chars": 1080, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2700.json", "ok": false, "error": null, "completion_tokens": 811, "reasoning_chars": 2204, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2257.json", "ok": false, "error": null, "completion_tokens": 279, "reasoning_chars": 424, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/14.json", "ok": false, "error": null, "completion_tokens": 189, "reasoning_chars": 474, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1300.json", "ok": false, "error": null, "completion_tokens": 736, "reasoning_chars": 1070, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/515.json", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 625, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1423.json", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1102.json", "ok": true, "error": null, "completion_tokens": 540, "reasoning_chars": 996, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1646.json", "ok": true, "error": null, "completion_tokens": 4379, "reasoning_chars": 11633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1272.json", "ok": false, "error": null, "completion_tokens": 153, "reasoning_chars": 262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/25149.json", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 445, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/697.json", "ok": true, "error": null, "completion_tokens": 1951, "reasoning_chars": 3601, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2036.json", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 299, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2037.json", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/353.json", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 638, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "bbh": { "task": "bbh", "n": 150, "hits": 72, "accuracy_pct": 48.0, "accuracy_ex_trunc_pct": 48.0, "truncated": 0, "completion_tokens_p50": 46, "completion_tokens_p90": 576, "completion_tokens_max": 10727, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1589, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 7919.7, "gen_tokens": 65087, "tok_per_s": 8.2, "details": [ { "id": "bbh-sports_understanding-101", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-60", "ok": true, "error": null, "completion_tokens": 476, "reasoning_chars": 1495, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-113", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-31", "ok": true, "error": null, "completion_tokens": 676, "reasoning_chars": 2396, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-144", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-28", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 583, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-116", "ok": true, "error": null, "completion_tokens": 53, "reasoning_chars": 147, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-67", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-3", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 1142, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-29", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-192", "ok": true, "error": null, "completion_tokens": 284, "reasoning_chars": 1050, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-196", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-10", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-83", "ok": true, "error": null, "completion_tokens": 394, "reasoning_chars": 1035, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-76", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-212", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 1174, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-87", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-1", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 924, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-140", "ok": true, "error": null, "completion_tokens": 546, "reasoning_chars": 2453, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-8", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 664, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-138", "ok": true, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-221", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-136", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-14", "ok": false, "error": null, "completion_tokens": 28, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-0", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-207", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-32", "ok": true, "error": null, "completion_tokens": 367, "reasoning_chars": 1589, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-83", "ok": true, "error": null, "completion_tokens": 406, "reasoning_chars": 1135, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-220", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-234", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-61", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-21", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-31", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-13", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-211", "ok": true, "error": null, "completion_tokens": 478, "reasoning_chars": 1476, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-37", "ok": true, "error": null, "completion_tokens": 217, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-225", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-90", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 54, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-101", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-179", "ok": true, "error": null, "completion_tokens": 336, "reasoning_chars": 1119, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-173", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 644, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-105", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-88", "ok": false, "error": null, "completion_tokens": 65, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-68", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-185", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-227", "ok": true, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-129", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-238", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 684, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-202", "ok": true, "error": null, "completion_tokens": 314, "reasoning_chars": 528, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-93", "ok": true, "error": null, "completion_tokens": 230, "reasoning_chars": 990, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-93", "ok": false, "error": null, "completion_tokens": 1131, "reasoning_chars": 4350, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-161", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-46", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-227", "ok": true, "error": null, "completion_tokens": 394, "reasoning_chars": 1349, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-158", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-205", "ok": true, "error": null, "completion_tokens": 451, "reasoning_chars": 1079, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-89", "ok": false, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-177", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-65", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 636, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-72", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-49", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 285, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-54", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-64", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-86", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-108", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 173, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-194", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-194", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-75", "ok": true, "error": null, "completion_tokens": 5290, "reasoning_chars": 18374, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-106", "ok": true, "error": null, "completion_tokens": 398, "reasoning_chars": 1239, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-81", "ok": false, "error": null, "completion_tokens": 5987, "reasoning_chars": 27415, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-242", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-150", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 874, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-25", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-139", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-12", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 1277, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-178", "ok": true, "error": null, "completion_tokens": 656, "reasoning_chars": 2580, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-160", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-115", "ok": true, "error": null, "completion_tokens": 5216, "reasoning_chars": 22632, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-44", "ok": true, "error": null, "completion_tokens": 675, "reasoning_chars": 2080, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-189", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 1365, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-102", "ok": true, "error": null, "completion_tokens": 311, "reasoning_chars": 1111, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-38", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-141", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-38", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 90, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-168", "ok": true, "error": null, "completion_tokens": 556, "reasoning_chars": 1328, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-212", "ok": false, "error": null, "completion_tokens": 10727, "reasoning_chars": 26480, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-245", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 578, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-222", "ok": true, "error": null, "completion_tokens": 282, "reasoning_chars": 1049, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-244", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 916, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-35", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-126", "ok": false, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-90", "ok": false, "error": null, "completion_tokens": 53, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-5", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-241", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-234", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 930, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-233", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-5", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-152", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-9", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-209", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-171", "ok": true, "error": null, "completion_tokens": 327, "reasoning_chars": 946, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-160", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-166", "ok": false, "error": null, "completion_tokens": 2634, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-213", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-240", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 77, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-95", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-227", "ok": true, "error": null, "completion_tokens": 496, "reasoning_chars": 1627, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-96", "ok": true, "error": null, "completion_tokens": 615, "reasoning_chars": 2650, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-33", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-4", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-180", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-74", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 961, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-88", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-10", "ok": true, "error": null, "completion_tokens": 221, "reasoning_chars": 811, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-191", "ok": false, "error": null, "completion_tokens": 3542, "reasoning_chars": 14420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-71", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-150", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-188", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 251, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-191", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 437, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-203", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 1090, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-237", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 687, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-24", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-27", "ok": true, "error": null, "completion_tokens": 440, "reasoning_chars": 1282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-177", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 973, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-192", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-242", "ok": false, "error": null, "completion_tokens": 56, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-51", "ok": false, "error": null, "completion_tokens": 7450, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-96", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-100", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-204", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-58", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 1762, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-157", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-104", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 621, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-47", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-150", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-23", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-68", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-174", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-108", "ok": true, "error": null, "completion_tokens": 1233, "reasoning_chars": 4270, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-203", "ok": false, "error": null, "completion_tokens": 402, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-115", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-223", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-221", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-5", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-233", "ok": true, "error": null, "completion_tokens": 891, "reasoning_chars": 628, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-3", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 1025, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-119", "ok": true, "error": null, "completion_tokens": 173, "reasoning_chars": 518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-197", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-103", "ok": true, "error": null, "completion_tokens": 1031, "reasoning_chars": 4489, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-178", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "humaneval": { "task": "humaneval", "n": 164, "hits": 149, "accuracy_pct": 90.85, "accuracy_ex_trunc_pct": 91.41, "truncated": 1, "completion_tokens_p50": 488, "completion_tokens_p90": 1470, "completion_tokens_max": 32768, "reasoning_chars_p50": 1023, "reasoning_chars_p90": 3191, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 11369.1, "gen_tokens": 208941, "tok_per_s": 18.4, "details": [ { "id": "HumanEval/0", "ok": true, "error": null, "completion_tokens": 667, "reasoning_chars": 2166, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/1", "ok": true, "error": null, "completion_tokens": 498, "reasoning_chars": 1653, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/2", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/3", "ok": true, "error": null, "completion_tokens": 314, "reasoning_chars": 1264, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/4", "ok": false, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/5", "ok": false, "error": null, "completion_tokens": 653, "reasoning_chars": 2358, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/6", "ok": false, "error": null, "completion_tokens": 625, "reasoning_chars": 2585, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/7", "ok": true, "error": null, "completion_tokens": 296, "reasoning_chars": 890, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/8", "ok": false, "error": null, "completion_tokens": 50, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/9", "ok": true, "error": null, "completion_tokens": 594, "reasoning_chars": 2477, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/10", "ok": false, "error": null, "completion_tokens": 866, "reasoning_chars": 3064, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/11", "ok": true, "error": null, "completion_tokens": 884, "reasoning_chars": 2902, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/12", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/13", "ok": true, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/14", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 746, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/15", "ok": true, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/16", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/17", "ok": true, "error": null, "completion_tokens": 615, "reasoning_chars": 1819, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/18", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/19", "ok": true, "error": null, "completion_tokens": 420, "reasoning_chars": 1318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/20", "ok": false, "error": null, "completion_tokens": 771, "reasoning_chars": 3061, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/21", "ok": false, "error": null, "completion_tokens": 2190, "reasoning_chars": 7709, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/22", "ok": true, "error": null, "completion_tokens": 1829, "reasoning_chars": 7699, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/23", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/24", "ok": true, "error": null, "completion_tokens": 853, "reasoning_chars": 2618, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/25", "ok": true, "error": null, "completion_tokens": 1349, "reasoning_chars": 4383, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/26", "ok": true, "error": null, "completion_tokens": 488, "reasoning_chars": 1894, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/27", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/28", "ok": true, "error": null, "completion_tokens": 252, "reasoning_chars": 830, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/29", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 766, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/30", "ok": true, "error": null, "completion_tokens": 234, "reasoning_chars": 462, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/31", "ok": true, "error": null, "completion_tokens": 456, "reasoning_chars": 981, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/32", "ok": false, "error": null, "completion_tokens": 9935, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/33", "ok": true, "error": null, "completion_tokens": 519, "reasoning_chars": 1023, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/34", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 598, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/35", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/36", "ok": true, "error": null, "completion_tokens": 1061, "reasoning_chars": 2437, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/37", "ok": true, "error": null, "completion_tokens": 724, "reasoning_chars": 1991, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/38", "ok": false, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/39", "ok": true, "error": null, "completion_tokens": 15036, "reasoning_chars": 39762, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/40", "ok": true, "error": null, "completion_tokens": 515, "reasoning_chars": 1095, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/41", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 1089, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/42", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/43", "ok": true, "error": null, "completion_tokens": 526, "reasoning_chars": 1203, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/44", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 719, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/45", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/46", "ok": true, "error": null, "completion_tokens": 692, "reasoning_chars": 944, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/47", "ok": true, "error": null, "completion_tokens": 69, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/48", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 397, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/49", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/50", "ok": false, "error": null, "completion_tokens": 273, "reasoning_chars": 836, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/51", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/52", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/53", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/54", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 706, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/55", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/56", "ok": true, "error": null, "completion_tokens": 328, "reasoning_chars": 749, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/57", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/58", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 660, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/59", "ok": false, "error": null, "completion_tokens": 1121, "reasoning_chars": 2891, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/60", "ok": true, "error": null, "completion_tokens": 680, "reasoning_chars": 2011, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/61", "ok": true, "error": null, "completion_tokens": 286, "reasoning_chars": 952, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/62", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 386, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/63", "ok": true, "error": null, "completion_tokens": 770, "reasoning_chars": 1560, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/64", "ok": true, "error": null, "completion_tokens": 650, "reasoning_chars": 1735, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/65", "ok": true, "error": null, "completion_tokens": 1048, "reasoning_chars": 3184, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/66", "ok": true, "error": null, "completion_tokens": 1372, "reasoning_chars": 4297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/67", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/68", "ok": true, "error": null, "completion_tokens": 598, "reasoning_chars": 758, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/69", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 1083, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/70", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 912, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/71", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/72", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 508, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/73", "ok": true, "error": null, "completion_tokens": 474, "reasoning_chars": 932, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/74", "ok": true, "error": null, "completion_tokens": 659, "reasoning_chars": 1515, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/75", "ok": true, "error": null, "completion_tokens": 3077, "reasoning_chars": 9362, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/76", "ok": true, "error": null, "completion_tokens": 7050, "reasoning_chars": 20747, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/77", "ok": true, "error": null, "completion_tokens": 1522, "reasoning_chars": 3804, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/78", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/79", "ok": true, "error": null, "completion_tokens": 343, "reasoning_chars": 652, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/80", "ok": true, "error": null, "completion_tokens": 76, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/81", "ok": true, "error": null, "completion_tokens": 1079, "reasoning_chars": 1501, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/82", "ok": true, "error": null, "completion_tokens": 335, "reasoning_chars": 774, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/83", "ok": true, "error": null, "completion_tokens": 925, "reasoning_chars": 2338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/84", "ok": true, "error": null, "completion_tokens": 315, "reasoning_chars": 483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/85", "ok": true, "error": null, "completion_tokens": 273, "reasoning_chars": 748, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/86", "ok": true, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/87", "ok": true, "error": null, "completion_tokens": 653, "reasoning_chars": 1306, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/88", "ok": true, "error": null, "completion_tokens": 566, "reasoning_chars": 1171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/89", "ok": true, "error": null, "completion_tokens": 576, "reasoning_chars": 1167, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/90", "ok": true, "error": null, "completion_tokens": 391, "reasoning_chars": 1025, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/91", "ok": true, "error": null, "completion_tokens": 725, "reasoning_chars": 2198, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/92", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/93", "ok": true, "error": null, "completion_tokens": 1177, "reasoning_chars": 3928, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/94", "ok": true, "error": null, "completion_tokens": 1470, "reasoning_chars": 2876, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/95", "ok": true, "error": null, "completion_tokens": 559, "reasoning_chars": 1956, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/96", "ok": true, "error": null, "completion_tokens": 994, "reasoning_chars": 2446, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/97", "ok": true, "error": null, "completion_tokens": 220, "reasoning_chars": 624, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/98", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 635, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/99", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 42156, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/100", "ok": true, "error": null, "completion_tokens": 619, "reasoning_chars": 1609, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/101", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/102", "ok": true, "error": null, "completion_tokens": 3471, "reasoning_chars": 10426, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/103", "ok": true, "error": null, "completion_tokens": 839, "reasoning_chars": 2107, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/104", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 649, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/105", "ok": true, "error": null, "completion_tokens": 104, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/106", "ok": true, "error": null, "completion_tokens": 657, "reasoning_chars": 1454, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/107", "ok": true, "error": null, "completion_tokens": 550, "reasoning_chars": 832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/108", "ok": true, "error": null, "completion_tokens": 927, "reasoning_chars": 2239, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/109", "ok": true, "error": null, "completion_tokens": 896, "reasoning_chars": 1798, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/110", "ok": true, "error": null, "completion_tokens": 814, "reasoning_chars": 2036, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/111", "ok": true, "error": null, "completion_tokens": 682, "reasoning_chars": 1908, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/112", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 601, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/113", "ok": true, "error": null, "completion_tokens": 1087, "reasoning_chars": 2838, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/114", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/115", "ok": true, "error": null, "completion_tokens": 526, "reasoning_chars": 669, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/116", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/117", "ok": true, "error": null, "completion_tokens": 627, "reasoning_chars": 1390, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/118", "ok": true, "error": null, "completion_tokens": 746, "reasoning_chars": 2012, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/119", "ok": true, "error": null, "completion_tokens": 771, "reasoning_chars": 1930, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/120", "ok": true, "error": null, "completion_tokens": 478, "reasoning_chars": 833, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/121", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 546, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/122", "ok": true, "error": null, "completion_tokens": 474, "reasoning_chars": 1127, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/123", "ok": true, "error": null, "completion_tokens": 1156, "reasoning_chars": 2823, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/124", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/125", "ok": true, "error": null, "completion_tokens": 70, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/126", "ok": true, "error": null, "completion_tokens": 898, "reasoning_chars": 2203, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/127", "ok": true, "error": null, "completion_tokens": 688, "reasoning_chars": 1931, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/128", "ok": true, "error": null, "completion_tokens": 553, "reasoning_chars": 1255, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/129", "ok": true, "error": null, "completion_tokens": 2998, "reasoning_chars": 10153, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/130", "ok": false, "error": null, "completion_tokens": 5698, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/131", "ok": true, "error": null, "completion_tokens": 414, "reasoning_chars": 1079, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/132", "ok": true, "error": null, "completion_tokens": 24603, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/133", "ok": true, "error": null, "completion_tokens": 786, "reasoning_chars": 2176, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/134", "ok": true, "error": null, "completion_tokens": 2303, "reasoning_chars": 7582, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/135", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 1132, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/136", "ok": true, "error": null, "completion_tokens": 669, "reasoning_chars": 1809, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/137", "ok": true, "error": null, "completion_tokens": 747, "reasoning_chars": 2197, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/138", "ok": true, "error": null, "completion_tokens": 280, "reasoning_chars": 653, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/139", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 643, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/140", "ok": true, "error": null, "completion_tokens": 813, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/141", "ok": true, "error": null, "completion_tokens": 105, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/142", "ok": true, "error": null, "completion_tokens": 817, "reasoning_chars": 1733, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/143", "ok": true, "error": null, "completion_tokens": 452, "reasoning_chars": 823, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/144", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 980, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/145", "ok": false, "error": null, "completion_tokens": 22818, "reasoning_chars": 80104, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/146", "ok": true, "error": null, "completion_tokens": 1208, "reasoning_chars": 3191, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/147", "ok": true, "error": null, "completion_tokens": 2309, "reasoning_chars": 5398, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/148", "ok": true, "error": null, "completion_tokens": 719, "reasoning_chars": 1658, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/149", "ok": true, "error": null, "completion_tokens": 547, "reasoning_chars": 1414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/150", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 788, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/151", "ok": true, "error": null, "completion_tokens": 763, "reasoning_chars": 2123, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/152", "ok": true, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/153", "ok": true, "error": null, "completion_tokens": 552, "reasoning_chars": 1882, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/154", "ok": true, "error": null, "completion_tokens": 363, "reasoning_chars": 775, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/155", "ok": true, "error": null, "completion_tokens": 578, "reasoning_chars": 1598, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/156", "ok": true, "error": null, "completion_tokens": 543, "reasoning_chars": 840, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/157", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/158", "ok": true, "error": null, "completion_tokens": 568, "reasoning_chars": 1687, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/159", "ok": true, "error": null, "completion_tokens": 448, "reasoning_chars": 500, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/160", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/161", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/162", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 1482, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/163", "ok": true, "error": null, "completion_tokens": 3621, "reasoning_chars": 10942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ifeval": { "task": "ifeval", "n": 100, "hits": 63, "accuracy_pct": 63.0, "accuracy_ex_trunc_pct": 64.29, "truncated": 2, "completion_tokens_p50": 429, "completion_tokens_p90": 2408, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 4262, "reasoning_leaked": null, "empty": 2, "length_limited": 2, "errors": 0, "elapsed_s": 10560.6, "gen_tokens": 162715, "tok_per_s": 15.4, "instruction_level_pct": 74.67, "instructions_checked": 150, "details": [ { "id": "ifeval-153", "ok": false, "error": null, "completion_tokens": 570, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-258", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-368", "ok": true, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-85", "ok": true, "error": null, "completion_tokens": 561, "reasoning_chars": 1845, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-246", "ok": false, "error": null, "completion_tokens": 173, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-23", "ok": true, "error": null, "completion_tokens": 3319, "reasoning_chars": 10375, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-372", "ok": true, "error": null, "completion_tokens": 1394, "reasoning_chars": 3595, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-387", "ok": true, "error": null, "completion_tokens": 972, "reasoning_chars": 3450, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-138", "ok": true, "error": null, "completion_tokens": 515, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-56", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-277", "ok": true, "error": null, "completion_tokens": 1205, "reasoning_chars": 3823, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-407", "ok": false, "error": null, "completion_tokens": 564, "reasoning_chars": 377, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-171", "ok": true, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-96", "ok": true, "error": null, "completion_tokens": 18399, "reasoning_chars": 19495, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-134", "ok": false, "error": null, "completion_tokens": 849, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-218", "ok": true, "error": null, "completion_tokens": 1154, "reasoning_chars": 3668, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-330", "ok": false, "error": null, "completion_tokens": 2475, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-6", "ok": true, "error": null, "completion_tokens": 502, "reasoning_chars": 1307, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-143", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 429, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-121", "ok": true, "error": null, "completion_tokens": 442, "reasoning_chars": 1389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-219", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-524", "ok": true, "error": null, "completion_tokens": 751, "reasoning_chars": 793, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-206", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-144", "ok": true, "error": null, "completion_tokens": 530, "reasoning_chars": 1188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-447", "ok": false, "error": null, "completion_tokens": 276, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-492", "ok": true, "error": null, "completion_tokens": 2231, "reasoning_chars": 6727, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-485", "ok": true, "error": null, "completion_tokens": 2408, "reasoning_chars": 5909, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-220", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-412", "ok": false, "error": null, "completion_tokens": 1443, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-342", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 390, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-222", "ok": true, "error": null, "completion_tokens": 450, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-131", "ok": false, "error": null, "completion_tokens": 217, "reasoning_chars": 633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-210", "ok": false, "error": null, "completion_tokens": 138, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-137", "ok": true, "error": null, "completion_tokens": 429, "reasoning_chars": 599, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-464", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-503", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-446", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 1246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-2", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-525", "ok": true, "error": null, "completion_tokens": 129, "reasoning_chars": 307, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-27", "ok": true, "error": null, "completion_tokens": 88, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-136", "ok": true, "error": null, "completion_tokens": 951, "reasoning_chars": 1767, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-286", "ok": false, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-449", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 1036, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-55", "ok": true, "error": null, "completion_tokens": 646, "reasoning_chars": 2051, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-93", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-399", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-19", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-229", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-181", "ok": true, "error": null, "completion_tokens": 638, "reasoning_chars": 2027, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-362", "ok": true, "error": null, "completion_tokens": 120, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-289", "ok": true, "error": null, "completion_tokens": 577, "reasoning_chars": 1831, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-356", "ok": true, "error": null, "completion_tokens": 714, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-115", "ok": false, "error": null, "completion_tokens": 358, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 1, "skipped": false }, { "id": "ifeval-404", "ok": false, "error": null, "completion_tokens": 724, "reasoning_chars": 1272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-243", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 81920, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-90", "ok": false, "error": null, "completion_tokens": 788, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-54", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-276", "ok": true, "error": null, "completion_tokens": 235, "reasoning_chars": 246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-308", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 610, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-64", "ok": false, "error": null, "completion_tokens": 1215, "reasoning_chars": 4262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-260", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-337", "ok": false, "error": null, "completion_tokens": 826, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-510", "ok": true, "error": null, "completion_tokens": 854, "reasoning_chars": 3383, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-119", "ok": true, "error": null, "completion_tokens": 1817, "reasoning_chars": 5508, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-5", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 825, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-468", "ok": false, "error": null, "completion_tokens": 451, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-418", "ok": false, "error": null, "completion_tokens": 591, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-476", "ok": true, "error": null, "completion_tokens": 2313, "reasoning_chars": 3914, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-49", "ok": true, "error": null, "completion_tokens": 6619, "reasoning_chars": 14592, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-336", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-422", "ok": true, "error": null, "completion_tokens": 131, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-417", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 782, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-347", "ok": false, "error": null, "completion_tokens": 235, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-101", "ok": true, "error": null, "completion_tokens": 624, "reasoning_chars": 1180, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-209", "ok": false, "error": null, "completion_tokens": 430, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-341", "ok": false, "error": null, "completion_tokens": 551, "reasoning_chars": 1337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-533", "ok": true, "error": null, "completion_tokens": 190, "reasoning_chars": 557, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-158", "ok": true, "error": null, "completion_tokens": 111, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-321", "ok": true, "error": null, "completion_tokens": 1215, "reasoning_chars": 5029, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-311", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-50", "ok": false, "error": null, "completion_tokens": 9826, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-518", "ok": true, "error": null, "completion_tokens": 3940, "reasoning_chars": 6347, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-374", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 871, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-317", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 573, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-512", "ok": true, "error": null, "completion_tokens": 977, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-102", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-72", "ok": false, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-33", "ok": false, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 1, "skipped": false }, { "id": "ifeval-263", "ok": false, "error": null, "completion_tokens": 1120, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-501", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 76793, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-4", "ok": true, "error": null, "completion_tokens": 343, "reasoning_chars": 1184, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-444", "ok": false, "error": null, "completion_tokens": 6348, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-228", "ok": true, "error": null, "completion_tokens": 267, "reasoning_chars": 186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-299", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-278", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-154", "ok": false, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 1, "skipped": false }, { "id": "ifeval-301", "ok": true, "error": null, "completion_tokens": 618, "reasoning_chars": 1996, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-161", "ok": false, "error": null, "completion_tokens": 3233, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-394", "ok": false, "error": null, "completion_tokens": 149, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-159", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 478, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false } ] } } }, "ablated-gguf": { "started": "2026-10-10 18:55:48", "finished": "2026-10-10 22:10:40", "seed": 20260816, "thinking": true, "tasks": { "mmlu": { "task": "mmlu", "n": 150, "hits": 102, "accuracy_pct": 68.0, "accuracy_ex_trunc_pct": 68.0, "truncated": 0, "completion_tokens_p50": 46, "completion_tokens_p90": 513, "completion_tokens_max": 20591, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1434, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 7938.9, "gen_tokens": 62395, "tok_per_s": 7.9, "details": [ { "id": "mmlu-12325", "ok": true, "error": null, "completion_tokens": 1129, "reasoning_chars": 3046, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-154", "ok": false, "error": null, "completion_tokens": 269, "reasoning_chars": 842, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12844", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1153", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11089", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12782", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5422", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12363", "ok": true, "error": null, "completion_tokens": 1810, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12556", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13163", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-270", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6943", "ok": true, "error": null, "completion_tokens": 1201, "reasoning_chars": 5199, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8330", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9276", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7584", "ok": true, "error": null, "completion_tokens": 137, "reasoning_chars": 367, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12819", "ok": true, "error": null, "completion_tokens": 472, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6163", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 408, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4099", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 918, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10389", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1539", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12013", "ok": true, "error": null, "completion_tokens": 1239, "reasoning_chars": 4554, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-7814", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1106", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 160, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1616", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 578, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-555", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 1013, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3698", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-618", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 566, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9842", "ok": true, "error": null, "completion_tokens": 205, "reasoning_chars": 590, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4229", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1022", "ok": true, "error": null, "completion_tokens": 273, "reasoning_chars": 1030, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6421", "ok": false, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1997", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 1148, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8941", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-565", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11364", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5959", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13066", "ok": false, "error": null, "completion_tokens": 476, "reasoning_chars": 1685, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3122", "ok": true, "error": null, "completion_tokens": 513, "reasoning_chars": 910, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3722", "ok": true, "error": null, "completion_tokens": 409, "reasoning_chars": 1411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13839", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12570", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9959", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 145, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10791", "ok": false, "error": null, "completion_tokens": 847, "reasoning_chars": 3997, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12268", "ok": true, "error": null, "completion_tokens": 1656, "reasoning_chars": 4391, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9051", "ok": false, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12887", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2667", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-3906", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-10304", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 115, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12927", "ok": true, "error": null, "completion_tokens": 232, "reasoning_chars": 703, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6337", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 98, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3907", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12888", "ok": false, "error": null, "completion_tokens": 464, "reasoning_chars": 1396, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10629", "ok": true, "error": null, "completion_tokens": 426, "reasoning_chars": 718, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5801", "ok": true, "error": null, "completion_tokens": 278, "reasoning_chars": 906, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2936", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7389", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9532", "ok": false, "error": null, "completion_tokens": 108, "reasoning_chars": 372, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8705", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 820, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11776", "ok": true, "error": null, "completion_tokens": 755, "reasoning_chars": 2216, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6836", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7856", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 512, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9697", "ok": true, "error": null, "completion_tokens": 469, "reasoning_chars": 1599, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10825", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11178", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11420", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-4658", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 575, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5403", "ok": false, "error": null, "completion_tokens": 397, "reasoning_chars": 1281, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6160", "ok": true, "error": null, "completion_tokens": 405, "reasoning_chars": 1073, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6872", "ok": false, "error": null, "completion_tokens": 10055, "reasoning_chars": 41490, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9738", "ok": true, "error": null, "completion_tokens": 47, "reasoning_chars": 173, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3076", "ok": false, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13496", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11289", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-222", "ok": false, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12180", "ok": true, "error": null, "completion_tokens": 901, "reasoning_chars": 2687, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1143", "ok": true, "error": null, "completion_tokens": 747, "reasoning_chars": 844, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13086", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-7038", "ok": true, "error": null, "completion_tokens": 276, "reasoning_chars": 834, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13587", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12349", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 720, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11747", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9848", "ok": true, "error": null, "completion_tokens": 848, "reasoning_chars": 2894, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3438", "ok": true, "error": null, "completion_tokens": 278, "reasoning_chars": 1128, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-89", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-4253", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 95, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-5460", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2494", "ok": false, "error": null, "completion_tokens": 20591, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11149", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-718", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5330", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-10716", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5636", "ok": false, "error": null, "completion_tokens": 817, "reasoning_chars": 2431, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11832", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-95", "ok": true, "error": null, "completion_tokens": 354, "reasoning_chars": 1114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13960", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2072", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12030", "ok": true, "error": null, "completion_tokens": 1474, "reasoning_chars": 5516, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2931", "ok": true, "error": null, "completion_tokens": 441, "reasoning_chars": 957, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2441", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13666", "ok": false, "error": null, "completion_tokens": 140, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6575", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-11625", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13283", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-4262", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12931", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12150", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-3678", "ok": true, "error": null, "completion_tokens": 324, "reasoning_chars": 1151, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4593", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1563", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13396", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-8049", "ok": true, "error": null, "completion_tokens": 79, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-8396", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 982, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-8493", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 1434, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13514", "ok": true, "error": null, "completion_tokens": 474, "reasoning_chars": 1420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-3000", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 136, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1635", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 548, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-1391", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-627", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1573", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 266, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6731", "ok": true, "error": null, "completion_tokens": 347, "reasoning_chars": 981, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11665", "ok": true, "error": null, "completion_tokens": 1130, "reasoning_chars": 3955, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-2914", "ok": false, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-6875", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2184", "ok": true, "error": null, "completion_tokens": 228, "reasoning_chars": 380, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12801", "ok": true, "error": null, "completion_tokens": 108, "reasoning_chars": 273, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-6717", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 488, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12464", "ok": false, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2787", "ok": true, "error": null, "completion_tokens": 428, "reasoning_chars": 1045, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11549", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-842", "ok": true, "error": null, "completion_tokens": 458, "reasoning_chars": 2134, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13868", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-9647", "ok": true, "error": null, "completion_tokens": 355, "reasoning_chars": 1375, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9502", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-12441", "ok": true, "error": null, "completion_tokens": 409, "reasoning_chars": 1338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-4392", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-5036", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 662, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12606", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-1400", "ok": true, "error": null, "completion_tokens": 81, "reasoning_chars": 371, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-13979", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-13634", "ok": true, "error": null, "completion_tokens": 205, "reasoning_chars": 952, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-11226", "ok": true, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-625", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-257", "ok": true, "error": null, "completion_tokens": 505, "reasoning_chars": 1240, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-12853", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 563, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-451", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlu-2074", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 41, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-9117", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 722, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlu-10748", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "cmmlu": { "task": "cmmlu", "n": 150, "hits": 97, "accuracy_pct": 64.67, "accuracy_ex_trunc_pct": 64.67, "truncated": 0, "completion_tokens_p50": 44, "completion_tokens_p90": 388, "completion_tokens_max": 2764, "reasoning_chars_p50": 0, "reasoning_chars_p90": 697, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 8282.3, "gen_tokens": 24182, "tok_per_s": 2.9, "details": [ { "id": "cmmlu-chinese_teacher_qualification-63", "ok": true, "error": null, "completion_tokens": 185, "reasoning_chars": 297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-54", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 254, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-91", "ok": false, "error": null, "completion_tokens": 333, "reasoning_chars": 420, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-260", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-32", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-29", "ok": true, "error": null, "completion_tokens": 68, "reasoning_chars": 54, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-200", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-98", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-44", "ok": true, "error": null, "completion_tokens": 218, "reasoning_chars": 344, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-118", "ok": true, "error": null, "completion_tokens": 231, "reasoning_chars": 762, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-116", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-43", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 530, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-145", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-247", "ok": true, "error": null, "completion_tokens": 176, "reasoning_chars": 308, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-224", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 285, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-26", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-105", "ok": true, "error": null, "completion_tokens": 1221, "reasoning_chars": 1753, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-15", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-62", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-nutrition-74", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-181", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-127", "ok": true, "error": null, "completion_tokens": 518, "reasoning_chars": 911, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_geography-107", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-18", "ok": true, "error": null, "completion_tokens": 332, "reasoning_chars": 582, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-82", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 160, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_food_culture-38", "ok": true, "error": null, "completion_tokens": 117, "reasoning_chars": 164, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-198", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 274, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-96", "ok": true, "error": null, "completion_tokens": 172, "reasoning_chars": 251, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-73", "ok": true, "error": null, "completion_tokens": 613, "reasoning_chars": 711, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-20", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-95", "ok": true, "error": null, "completion_tokens": 123, "reasoning_chars": 197, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-88", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-192", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 697, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-66", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 233, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-100", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 406, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-136", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-16", "ok": false, "error": null, "completion_tokens": 274, "reasoning_chars": 1045, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-logical-90", "ok": true, "error": null, "completion_tokens": 513, "reasoning_chars": 1661, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-41", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-345", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-103", "ok": false, "error": null, "completion_tokens": 135, "reasoning_chars": 217, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-93", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-88", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-206", "ok": true, "error": null, "completion_tokens": 183, "reasoning_chars": 303, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-105", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-90", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-190", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-international_law-39", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-155", "ok": true, "error": null, "completion_tokens": 959, "reasoning_chars": 3310, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-24", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-6", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-31", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-243", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-63", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-135", "ok": false, "error": null, "completion_tokens": 189, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-89", "ok": true, "error": null, "completion_tokens": 60, "reasoning_chars": 82, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-91", "ok": false, "error": null, "completion_tokens": 117, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-120", "ok": true, "error": null, "completion_tokens": 619, "reasoning_chars": 1065, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-62", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-106", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-food_science-106", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-24", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-157", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-41", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-115", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-journalism-29", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 110, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-traditional_chinese_medicine-67", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 95, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-161", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 159, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-120", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-education-117", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 212, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-169", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-129", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-19", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-8", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-72", "ok": true, "error": null, "completion_tokens": 298, "reasoning_chars": 575, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-139", "ok": false, "error": null, "completion_tokens": 206, "reasoning_chars": 681, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_security-22", "ok": true, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-7", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_law-108", "ok": true, "error": null, "completion_tokens": 109, "reasoning_chars": 165, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-31", "ok": true, "error": null, "completion_tokens": 28, "reasoning_chars": 31, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-131", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-35", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_actuarial_science-38", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-165", "ok": true, "error": null, "completion_tokens": 153, "reasoning_chars": 215, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-marketing-101", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-107", "ok": true, "error": null, "completion_tokens": 84, "reasoning_chars": 105, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-8", "ok": false, "error": null, "completion_tokens": 2764, "reasoning_chars": 11074, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medical_statistics-86", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-security_study-30", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 59, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-clinical_knowledge-118", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_law-24", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-46", "ok": true, "error": null, "completion_tokens": 618, "reasoning_chars": 2018, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-anatomy-21", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-145", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-91", "ok": true, "error": null, "completion_tokens": 64, "reasoning_chars": 84, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_chinese-109", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 9, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-machine_learning-44", "ok": false, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-74", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_mathematics-63", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-146", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-109", "ok": true, "error": null, "completion_tokens": 342, "reasoning_chars": 633, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-construction_project_management-16", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-marxist_theory-113", "ok": false, "error": null, "completion_tokens": 163, "reasoning_chars": 285, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-102", "ok": false, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-207", "ok": false, "error": null, "completion_tokens": 53, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-19", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-164", "ok": true, "error": null, "completion_tokens": 227, "reasoning_chars": 399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-338", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 202, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-7", "ok": true, "error": null, "completion_tokens": 601, "reasoning_chars": 1653, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-92", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-47", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 1171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-astronomy-140", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-legal_and_moral_basis-100", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 141, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-37", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_commonsense-191", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_medicine-34", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-88", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-public_relations-77", "ok": false, "error": null, "completion_tokens": 74, "reasoning_chars": 110, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_history-306", "ok": true, "error": null, "completion_tokens": 748, "reasoning_chars": 2021, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-business_ethics-40", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 149, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_chemistry-7", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-152", "ok": true, "error": null, "completion_tokens": 200, "reasoning_chars": 339, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-1", "ok": true, "error": null, "completion_tokens": 124, "reasoning_chars": 178, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-94", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_psychology-109", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 257, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_literature-164", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 174, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-sports_science-89", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-economics-17", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-arts-2", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 212, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-51", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-agronomy-141", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 279, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-computer_science-196", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-genetics-97", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-jurisprudence-202", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 664, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-management-195", "ok": true, "error": null, "completion_tokens": 336, "reasoning_chars": 253, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-conceptual_physics-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-sociology-49", "ok": true, "error": null, "completion_tokens": 63, "reasoning_chars": 101, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_engineering_hydrology-94", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-ancient_chinese-8", "ok": false, "error": null, "completion_tokens": 506, "reasoning_chars": 629, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_medicine-98", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-modern_chinese-28", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_foreign_policy-74", "ok": false, "error": null, "completion_tokens": 53, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_driving_rule-121", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 663, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_mathematics-120", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-elementary_information_and_technology-105", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 113, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-chinese_teacher_qualification-35", "ok": true, "error": null, "completion_tokens": 181, "reasoning_chars": 215, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-professional_accounting-51", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "cmmlu-college_mathematics-49", "ok": true, "error": null, "completion_tokens": 1992, "reasoning_chars": 4739, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-high_school_politics-130", "ok": true, "error": null, "completion_tokens": 470, "reasoning_chars": 1478, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "cmmlu-virology-140", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "mmlu_pro": { "task": "mmlu_pro", "n": 150, "hits": 103, "accuracy_pct": 68.67, "accuracy_ex_trunc_pct": 69.13, "truncated": 1, "completion_tokens_p50": 67, "completion_tokens_p90": 1654, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 3678, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 3046.9, "gen_tokens": 137876, "tok_per_s": 45.3, "details": [ { "id": "mmlupro-5831", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2299", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9280", "ok": true, "error": null, "completion_tokens": 152, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5486", "ok": true, "error": null, "completion_tokens": 521, "reasoning_chars": 2383, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5761", "ok": true, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6589", "ok": true, "error": null, "completion_tokens": 881, "reasoning_chars": 2013, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5389", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5131", "ok": false, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1677", "ok": true, "error": null, "completion_tokens": 1254, "reasoning_chars": 4343, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4388", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 46905, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "mmlupro-3285", "ok": true, "error": null, "completion_tokens": 1010, "reasoning_chars": 1506, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4118", "ok": true, "error": null, "completion_tokens": 900, "reasoning_chars": 2304, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5344", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11696", "ok": false, "error": null, "completion_tokens": 1437, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-858", "ok": true, "error": null, "completion_tokens": 100, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-214", "ok": true, "error": null, "completion_tokens": 1044, "reasoning_chars": 1793, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6609", "ok": true, "error": null, "completion_tokens": 743, "reasoning_chars": 1828, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2083", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9752", "ok": false, "error": null, "completion_tokens": 6662, "reasoning_chars": 17528, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7213", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5234", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8570", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 293, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5612", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2071", "ok": true, "error": null, "completion_tokens": 683, "reasoning_chars": 2335, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9842", "ok": true, "error": null, "completion_tokens": 54, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4665", "ok": true, "error": null, "completion_tokens": 916, "reasoning_chars": 2200, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5564", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6475", "ok": true, "error": null, "completion_tokens": 413, "reasoning_chars": 900, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2153", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5812", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 141, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3407", "ok": true, "error": null, "completion_tokens": 2969, "reasoning_chars": 5560, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2879", "ok": false, "error": null, "completion_tokens": 287, "reasoning_chars": 423, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8266", "ok": true, "error": null, "completion_tokens": 389, "reasoning_chars": 516, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1916", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3486", "ok": false, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11275", "ok": true, "error": null, "completion_tokens": 357, "reasoning_chars": 999, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6040", "ok": true, "error": null, "completion_tokens": 1668, "reasoning_chars": 4756, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8565", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 261, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6848", "ok": false, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8926", "ok": true, "error": null, "completion_tokens": 249, "reasoning_chars": 297, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8458", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7854", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3962", "ok": true, "error": null, "completion_tokens": 3531, "reasoning_chars": 4305, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6470", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3813", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 1087, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8026", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11030", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8151", "ok": true, "error": null, "completion_tokens": 62, "reasoning_chars": 81, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5286", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3888", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11676", "ok": true, "error": null, "completion_tokens": 3148, "reasoning_chars": 5730, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8762", "ok": false, "error": null, "completion_tokens": 1654, "reasoning_chars": 2863, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-12160", "ok": true, "error": null, "completion_tokens": 563, "reasoning_chars": 657, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4022", "ok": true, "error": null, "completion_tokens": 2506, "reasoning_chars": 3196, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10092", "ok": true, "error": null, "completion_tokens": 598, "reasoning_chars": 1372, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7364", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6001", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10873", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9777", "ok": true, "error": null, "completion_tokens": 10547, "reasoning_chars": 23294, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8723", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4595", "ok": true, "error": null, "completion_tokens": 402, "reasoning_chars": 999, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8423", "ok": true, "error": null, "completion_tokens": 3254, "reasoning_chars": 4944, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11051", "ok": false, "error": null, "completion_tokens": 112, "reasoning_chars": 557, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-836", "ok": false, "error": null, "completion_tokens": 30, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11297", "ok": true, "error": null, "completion_tokens": 13993, "reasoning_chars": 36076, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3950", "ok": true, "error": null, "completion_tokens": 274, "reasoning_chars": 612, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1562", "ok": false, "error": null, "completion_tokens": 933, "reasoning_chars": 3678, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2041", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3473", "ok": true, "error": null, "completion_tokens": 286, "reasoning_chars": 1288, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11253", "ok": true, "error": null, "completion_tokens": 713, "reasoning_chars": 2502, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3858", "ok": true, "error": null, "completion_tokens": 548, "reasoning_chars": 1016, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2231", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2190", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6930", "ok": true, "error": null, "completion_tokens": 457, "reasoning_chars": 1712, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-6558", "ok": true, "error": null, "completion_tokens": 592, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5371", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-1766", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6045", "ok": true, "error": null, "completion_tokens": 1161, "reasoning_chars": 3051, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2298", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6400", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6499", "ok": true, "error": null, "completion_tokens": 553, "reasoning_chars": 1393, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7936", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 1000, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-8566", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11633", "ok": false, "error": null, "completion_tokens": 68, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9843", "ok": false, "error": null, "completion_tokens": 8115, "reasoning_chars": 20744, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11758", "ok": false, "error": null, "completion_tokens": 339, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2843", "ok": true, "error": null, "completion_tokens": 1253, "reasoning_chars": 3282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7483", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11379", "ok": true, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4866", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-818", "ok": false, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2785", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8119", "ok": false, "error": null, "completion_tokens": 216, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2214", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5774", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6049", "ok": false, "error": null, "completion_tokens": 422, "reasoning_chars": 1096, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7219", "ok": true, "error": null, "completion_tokens": 492, "reasoning_chars": 1803, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1660", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4767", "ok": true, "error": null, "completion_tokens": 106, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5964", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2743", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4062", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 637, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-7368", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-827", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11877", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7557", "ok": false, "error": null, "completion_tokens": 943, "reasoning_chars": 3302, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9998", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 612, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10098", "ok": true, "error": null, "completion_tokens": 761, "reasoning_chars": 1857, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1744", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9210", "ok": true, "error": null, "completion_tokens": 372, "reasoning_chars": 464, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11248", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 874, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11187", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2799", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4479", "ok": true, "error": null, "completion_tokens": 1894, "reasoning_chars": 4345, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10223", "ok": true, "error": null, "completion_tokens": 573, "reasoning_chars": 1294, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-5698", "ok": false, "error": null, "completion_tokens": 29, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8291", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3516", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5466", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4851", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-733", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-6995", "ok": true, "error": null, "completion_tokens": 48, "reasoning_chars": 169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2573", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8481", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-5938", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-10708", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8476", "ok": false, "error": null, "completion_tokens": 56, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-7990", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 742, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-2409", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11551", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-674", "ok": true, "error": null, "completion_tokens": 169, "reasoning_chars": 211, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-1978", "ok": true, "error": null, "completion_tokens": 1149, "reasoning_chars": 4029, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10420", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-3445", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9488", "ok": true, "error": null, "completion_tokens": 337, "reasoning_chars": 814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4993", "ok": true, "error": null, "completion_tokens": 454, "reasoning_chars": 1470, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4316", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 310, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4730", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-11570", "ok": false, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-2149", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8322", "ok": true, "error": null, "completion_tokens": 2791, "reasoning_chars": 7219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-10993", "ok": false, "error": null, "completion_tokens": 822, "reasoning_chars": 2421, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-4588", "ok": true, "error": null, "completion_tokens": 4721, "reasoning_chars": 10208, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-3258", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-8629", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-637", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-4257", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 518, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-9719", "ok": true, "error": null, "completion_tokens": 799, "reasoning_chars": 1236, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "mmlupro-11714", "ok": false, "error": null, "completion_tokens": 4819, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "mmlupro-9678", "ok": false, "error": null, "completion_tokens": 1001, "reasoning_chars": 3378, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ceval": { "task": "ceval", "n": 150, "hits": 105, "accuracy_pct": 70.0, "accuracy_ex_trunc_pct": 70.47, "truncated": 1, "completion_tokens_p50": 41, "completion_tokens_p90": 632, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 1483, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 7987.8, "gen_tokens": 80359, "tok_per_s": 10.1, "details": [ { "id": "ceval-high_school_chinese-1", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-10", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-41", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-3", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-21", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 175, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-4", "ok": true, "error": null, "completion_tokens": 1416, "reasoning_chars": 3209, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-21", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 257, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-2", "ok": true, "error": null, "completion_tokens": 80, "reasoning_chars": 124, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-6", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-12", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-19", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chemistry-9", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-15", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-22", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 22, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-16", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 54, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-11", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-18", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 158, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-24", "ok": true, "error": null, "completion_tokens": 587, "reasoning_chars": 1834, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-29", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-2", "ok": true, "error": null, "completion_tokens": 1204, "reasoning_chars": 1483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-11", "ok": false, "error": null, "completion_tokens": 66, "reasoning_chars": 114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-18", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-0", "ok": false, "error": null, "completion_tokens": 41, "reasoning_chars": 52, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-14", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-27", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-19", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 245, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-21", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-33", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 231, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-20", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-10", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 706, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-19", "ok": true, "error": null, "completion_tokens": 51, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-7", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 99, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-41", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-44", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 254, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-9", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-28", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-39", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-1", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-24", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-5", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-computer_architecture-15", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-39", "ok": true, "error": null, "completion_tokens": 400, "reasoning_chars": 1819, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-23", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-7", "ok": false, "error": null, "completion_tokens": 110, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-17", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 416, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-6", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-15", "ok": true, "error": null, "completion_tokens": 71, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-4", "ok": true, "error": null, "completion_tokens": 7936, "reasoning_chars": 21572, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-5", "ok": true, "error": null, "completion_tokens": 1039, "reasoning_chars": 3279, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-50", "ok": true, "error": null, "completion_tokens": 136, "reasoning_chars": 169, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_politics-11", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-32", "ok": false, "error": null, "completion_tokens": 455, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-logic-16", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-9", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-art_studies-15", "ok": true, "error": null, "completion_tokens": 55, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-2", "ok": false, "error": null, "completion_tokens": 9547, "reasoning_chars": 24634, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-7", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 263, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-11", "ok": true, "error": null, "completion_tokens": 36, "reasoning_chars": 45, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-operating_system-18", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 103, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-4", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-17", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-1", "ok": true, "error": null, "completion_tokens": 143, "reasoning_chars": 268, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-32", "ok": true, "error": null, "completion_tokens": 110, "reasoning_chars": 210, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-33", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-25", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-40", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 197, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-7", "ok": true, "error": null, "completion_tokens": 907, "reasoning_chars": 1215, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-5", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 1934, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-12", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-4", "ok": true, "error": null, "completion_tokens": 216, "reasoning_chars": 326, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-38", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 66990, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "ceval-electrical_engineer-22", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-clinical_medicine-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-15", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-45", "ok": true, "error": null, "completion_tokens": 91, "reasoning_chars": 103, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-6", "ok": false, "error": null, "completion_tokens": 40, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-legal_professional-20", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-2", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-17", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-16", "ok": false, "error": null, "completion_tokens": 443, "reasoning_chars": 1774, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-7", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_biology-16", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-10", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 943, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-32", "ok": true, "error": null, "completion_tokens": 5058, "reasoning_chars": 13264, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_geography-0", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-12", "ok": true, "error": null, "completion_tokens": 1004, "reasoning_chars": 1324, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-logic-14", "ok": true, "error": null, "completion_tokens": 711, "reasoning_chars": 2424, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-physician-26", "ok": true, "error": null, "completion_tokens": 632, "reasoning_chars": 578, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-3", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-3", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 18, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-computer_network-13", "ok": true, "error": null, "completion_tokens": 32, "reasoning_chars": 41, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-sports_science-5", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-electrical_engineer-27", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-fire_engineer-21", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-10", "ok": true, "error": null, "completion_tokens": 131, "reasoning_chars": 167, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-16", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-49", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-13", "ok": true, "error": null, "completion_tokens": 1090, "reasoning_chars": 3934, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-business_administration-5", "ok": false, "error": null, "completion_tokens": 130, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-2", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 58, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-27", "ok": false, "error": null, "completion_tokens": 333, "reasoning_chars": 646, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_biology-8", "ok": true, "error": null, "completion_tokens": 96, "reasoning_chars": 129, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_physics-14", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-14", "ok": true, "error": null, "completion_tokens": 132, "reasoning_chars": 182, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-31", "ok": true, "error": null, "completion_tokens": 568, "reasoning_chars": 1041, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-basic_medicine-13", "ok": true, "error": null, "completion_tokens": 52, "reasoning_chars": 73, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_programming-12", "ok": true, "error": null, "completion_tokens": 480, "reasoning_chars": 1257, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-civil_servant-44", "ok": true, "error": null, "completion_tokens": 1350, "reasoning_chars": 3916, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-3", "ok": true, "error": null, "completion_tokens": 1774, "reasoning_chars": 4638, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-13", "ok": true, "error": null, "completion_tokens": 800, "reasoning_chars": 814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-6", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_politics-19", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 1133, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-environmental_impact_assessment_engineer-27", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-law-10", "ok": true, "error": null, "completion_tokens": 252, "reasoning_chars": 315, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-law-0", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-3", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 254, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-modern_chinese_history-6", "ok": true, "error": null, "completion_tokens": 300, "reasoning_chars": 274, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_chinese-13", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-urban_and_rural_planner-43", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-42", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_mathematics-13", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-discrete_mathematics-1", "ok": true, "error": null, "completion_tokens": 51, "reasoning_chars": 65, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-11", "ok": true, "error": null, "completion_tokens": 853, "reasoning_chars": 1063, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_history-16", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 107, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-plant_protection-2", "ok": true, "error": null, "completion_tokens": 156, "reasoning_chars": 33, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-probability_and_statistics-1", "ok": true, "error": null, "completion_tokens": 262, "reasoning_chars": 618, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-16", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_chemistry-11", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-teacher_qualification-5", "ok": true, "error": null, "completion_tokens": 214, "reasoning_chars": 392, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-40", "ok": true, "error": null, "completion_tokens": 506, "reasoning_chars": 1631, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_physics-6", "ok": false, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-middle_school_mathematics-10", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-ideological_and_moral_cultivation-10", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-17", "ok": false, "error": null, "completion_tokens": 394, "reasoning_chars": 578, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-tax_accountant-11", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-professional_tour_guide-16", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 64, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-education_science-1", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_physics-0", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-37", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-9", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 18, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-high_school_history-16", "ok": true, "error": null, "completion_tokens": 260, "reasoning_chars": 353, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-chinese_language_and_literature-5", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-advanced_mathematics-18", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-physician-30", "ok": true, "error": null, "completion_tokens": 99, "reasoning_chars": 168, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-accountant-21", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 319, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "ceval-college_economics-42", "ok": false, "error": null, "completion_tokens": 73, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-veterinary_medicine-5", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "ceval-college_chemistry-4", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "arc": { "task": "arc", "n": 150, "hits": 137, "accuracy_pct": 91.33, "accuracy_ex_trunc_pct": 91.33, "truncated": 0, "completion_tokens_p50": 247, "completion_tokens_p90": 471, "completion_tokens_max": 2134, "reasoning_chars_p50": 763, "reasoning_chars_p90": 1429, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 8042.1, "gen_tokens": 40490, "tok_per_s": 5.0, "details": [ { "id": "arc-Mercury_415084", "ok": true, "error": null, "completion_tokens": 94, "reasoning_chars": 58, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7249708", "ok": true, "error": null, "completion_tokens": 437, "reasoning_chars": 1290, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_4_24", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 1080, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7056823", "ok": true, "error": null, "completion_tokens": 310, "reasoning_chars": 899, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415534", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 682, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405827", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263183", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 684, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106960", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_7", "ok": true, "error": null, "completion_tokens": 46, "reasoning_chars": 246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7182245", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 1692, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416177", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2011_5_8", "ok": true, "error": null, "completion_tokens": 346, "reasoning_chars": 1066, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_8_36", "ok": true, "error": null, "completion_tokens": 366, "reasoning_chars": 1059, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401699", "ok": true, "error": null, "completion_tokens": 103, "reasoning_chars": 298, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107240", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 290, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_4_26", "ok": true, "error": null, "completion_tokens": 321, "reasoning_chars": 728, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402638", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 1812, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026723", "ok": true, "error": null, "completion_tokens": 40, "reasoning_chars": 163, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415422", "ok": true, "error": null, "completion_tokens": 291, "reasoning_chars": 877, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2000_8_2", "ok": true, "error": null, "completion_tokens": 340, "reasoning_chars": 1065, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2006_9_4", "ok": true, "error": null, "completion_tokens": 626, "reasoning_chars": 1695, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401272", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 408, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416169", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 743, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7116235", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 1377, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7109463", "ok": true, "error": null, "completion_tokens": 246, "reasoning_chars": 609, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7032428", "ok": false, "error": null, "completion_tokens": 738, "reasoning_chars": 2238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2008_8_11", "ok": true, "error": null, "completion_tokens": 325, "reasoning_chars": 909, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_410334", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 1112, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_36", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_415719", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 496, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_17", "ok": true, "error": null, "completion_tokens": 154, "reasoning_chars": 763, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402256", "ok": true, "error": null, "completion_tokens": 92, "reasoning_chars": 345, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_400443", "ok": false, "error": null, "completion_tokens": 20, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7133245", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 693, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413458", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401372", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 728, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401116", "ok": true, "error": null, "completion_tokens": 222, "reasoning_chars": 664, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__5_10312", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 904, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2007_4_10280", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416683", "ok": true, "error": null, "completion_tokens": 453, "reasoning_chars": 1223, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7190120", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 910, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401295", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 1352, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_4_28", "ok": true, "error": null, "completion_tokens": 130, "reasoning_chars": 347, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7223493", "ok": true, "error": null, "completion_tokens": 319, "reasoning_chars": 1081, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018410", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 365, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1998_8_24", "ok": true, "error": null, "completion_tokens": 566, "reasoning_chars": 1703, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2003_8_31", "ok": true, "error": null, "completion_tokens": 320, "reasoning_chars": 903, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_405501", "ok": true, "error": null, "completion_tokens": 384, "reasoning_chars": 1087, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408030", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 1003, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401684", "ok": true, "error": null, "completion_tokens": 284, "reasoning_chars": 1256, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041668", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 977, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7205870", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 915, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7210140", "ok": true, "error": null, "completion_tokens": 474, "reasoning_chars": 1482, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405873", "ok": true, "error": null, "completion_tokens": 388, "reasoning_chars": 1312, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400052", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7018270", "ok": true, "error": null, "completion_tokens": 562, "reasoning_chars": 1732, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406550", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 772, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7150728", "ok": true, "error": null, "completion_tokens": 471, "reasoning_chars": 1311, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400839", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 681, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_30", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 807, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400383", "ok": false, "error": null, "completion_tokens": 480, "reasoning_chars": 1368, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7236740", "ok": true, "error": null, "completion_tokens": 208, "reasoning_chars": 300, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-CSZ_2005_5_CSZ10021", "ok": true, "error": null, "completion_tokens": 179, "reasoning_chars": 775, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7093958", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 792, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7026355", "ok": true, "error": null, "completion_tokens": 48, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408324", "ok": true, "error": null, "completion_tokens": 148, "reasoning_chars": 338, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP__7_10348", "ok": true, "error": null, "completion_tokens": 155, "reasoning_chars": 315, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402629", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 469, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400046", "ok": true, "error": null, "completion_tokens": 421, "reasoning_chars": 1141, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7180810", "ok": true, "error": null, "completion_tokens": 399, "reasoning_chars": 1081, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_180005", "ok": true, "error": null, "completion_tokens": 162, "reasoning_chars": 675, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AIMS_2009_8_14", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2007_5_16", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_LBS10662", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 352, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_23", "ok": true, "error": null, "completion_tokens": 233, "reasoning_chars": 756, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2012_8_19", "ok": false, "error": null, "completion_tokens": 118, "reasoning_chars": 524, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7041650", "ok": true, "error": null, "completion_tokens": 363, "reasoning_chars": 852, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416181", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 389, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_1995_8_J6", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 1099, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2014_5_13", "ok": true, "error": null, "completion_tokens": 365, "reasoning_chars": 1432, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7106750", "ok": true, "error": null, "completion_tokens": 381, "reasoning_chars": 1041, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_4_23", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 603, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MEA_2013_5_12", "ok": true, "error": null, "completion_tokens": 118, "reasoning_chars": 195, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_8_42", "ok": false, "error": null, "completion_tokens": 163, "reasoning_chars": 419, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_408367", "ok": true, "error": null, "completion_tokens": 142, "reasoning_chars": 376, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7250058", "ok": true, "error": null, "completion_tokens": 121, "reasoning_chars": 559, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_414041", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 304, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401290", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 1077, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409295", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 1112, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7211680", "ok": true, "error": null, "completion_tokens": 463, "reasoning_chars": 1659, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2009_5_37", "ok": true, "error": null, "completion_tokens": 358, "reasoning_chars": 1051, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413439", "ok": true, "error": null, "completion_tokens": 436, "reasoning_chars": 1275, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2013_8_32", "ok": true, "error": null, "completion_tokens": 283, "reasoning_chars": 677, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7216423", "ok": true, "error": null, "completion_tokens": 326, "reasoning_chars": 1022, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-ACTAAP_2015_7_9", "ok": true, "error": null, "completion_tokens": 53, "reasoning_chars": 179, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7230405", "ok": true, "error": null, "completion_tokens": 561, "reasoning_chars": 1694, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7025008", "ok": true, "error": null, "completion_tokens": 508, "reasoning_chars": 1445, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_416536", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 527, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TIMSS_2007_8_pg60", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_4_34", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 1429, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_406833", "ok": true, "error": null, "completion_tokens": 363, "reasoning_chars": 1034, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_409647", "ok": true, "error": null, "completion_tokens": 417, "reasoning_chars": 1146, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_184363", "ok": true, "error": null, "completion_tokens": 218, "reasoning_chars": 548, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7107170", "ok": true, "error": null, "completion_tokens": 287, "reasoning_chars": 736, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_416104", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 248, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_10-v1", "ok": true, "error": null, "completion_tokens": 75, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7202843", "ok": true, "error": null, "completion_tokens": 539, "reasoning_chars": 1445, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029803", "ok": true, "error": null, "completion_tokens": 373, "reasoning_chars": 1113, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7164623", "ok": true, "error": null, "completion_tokens": 268, "reasoning_chars": 889, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-TAKS_2009_5_26", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7263060", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 1065, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268328", "ok": true, "error": null, "completion_tokens": 281, "reasoning_chars": 778, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_402282", "ok": true, "error": null, "completion_tokens": 59, "reasoning_chars": 290, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7029558", "ok": true, "error": null, "completion_tokens": 50, "reasoning_chars": 205, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2013_4_21", "ok": true, "error": null, "completion_tokens": 146, "reasoning_chars": 397, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7186358", "ok": true, "error": null, "completion_tokens": 433, "reasoning_chars": 1327, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-LEAP_2011_8_10434", "ok": true, "error": null, "completion_tokens": 424, "reasoning_chars": 1159, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2012_8_23651", "ok": true, "error": null, "completion_tokens": 240, "reasoning_chars": 624, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_407125", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 577, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7268258", "ok": false, "error": null, "completion_tokens": 65, "reasoning_chars": 284, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7027405", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 775, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_8_7", "ok": true, "error": null, "completion_tokens": 315, "reasoning_chars": 1498, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_191625", "ok": true, "error": null, "completion_tokens": 2134, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2005_9_6", "ok": true, "error": null, "completion_tokens": 161, "reasoning_chars": 186, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7219678", "ok": true, "error": null, "completion_tokens": 259, "reasoning_chars": 1049, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_401678", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 249, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_401312", "ok": true, "error": null, "completion_tokens": 202, "reasoning_chars": 680, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2010_8_42", "ok": false, "error": null, "completion_tokens": 236, "reasoning_chars": 165, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7030468", "ok": true, "error": null, "completion_tokens": 404, "reasoning_chars": 1151, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175385", "ok": true, "error": null, "completion_tokens": 396, "reasoning_chars": 1132, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_175560", "ok": true, "error": null, "completion_tokens": 97, "reasoning_chars": 210, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_2004_9_5", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 137, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_409009", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 825, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7128923", "ok": false, "error": null, "completion_tokens": 689, "reasoning_chars": 2348, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-NYSEDREGENTS_2008_8_9", "ok": false, "error": null, "completion_tokens": 32, "reasoning_chars": 113, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7042945", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 1070, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-AKDE&ED_2008_8_47", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 993, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7185343", "ok": true, "error": null, "completion_tokens": 493, "reasoning_chars": 1645, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7218400", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7221988", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_405950", "ok": true, "error": null, "completion_tokens": 74, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7222530", "ok": true, "error": null, "completion_tokens": 385, "reasoning_chars": 1063, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7102323", "ok": true, "error": null, "completion_tokens": 177, "reasoning_chars": 358, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MDSA_2009_5_39", "ok": true, "error": null, "completion_tokens": 246, "reasoning_chars": 617, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-MCAS_1999_8_5", "ok": true, "error": null, "completion_tokens": 188, "reasoning_chars": 770, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_400661", "ok": true, "error": null, "completion_tokens": 413, "reasoning_chars": 1104, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_7081603", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 663, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-VASoL_2008_3_25", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 599, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_SC_413628", "ok": true, "error": null, "completion_tokens": 330, "reasoning_chars": 896, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "arc-Mercury_406427", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 1134, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "truthfulqa": { "task": "truthfulqa", "n": 150, "hits": 91, "accuracy_pct": 60.67, "accuracy_ex_trunc_pct": 60.67, "truncated": 0, "completion_tokens_p50": 7, "completion_tokens_p90": 210, "completion_tokens_max": 1012, "reasoning_chars_p50": 0, "reasoning_chars_p90": 782, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 8001.9, "gen_tokens": 8743, "tok_per_s": 1.1, "details": [ { "id": "tqa-210", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-76", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-461", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-218", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-254", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-635", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-193", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-148", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-400", "ok": false, "error": null, "completion_tokens": 281, "reasoning_chars": 999, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-766", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-240", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-50", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-83", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-381", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-325", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-655", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-721", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-391", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-401", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-625", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-80", "ok": false, "error": null, "completion_tokens": 325, "reasoning_chars": 786, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-439", "ok": true, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-66", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-155", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-516", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-522", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-47", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-343", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-126", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-77", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-613", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-590", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-674", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-438", "ok": false, "error": null, "completion_tokens": 106, "reasoning_chars": 415, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-503", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-639", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-111", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-289", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-705", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-373", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-363", "ok": true, "error": null, "completion_tokens": 1012, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-792", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-113", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-74", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-634", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 809, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-804", "ok": true, "error": null, "completion_tokens": 464, "reasoning_chars": 1729, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-626", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-676", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-174", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-455", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-725", "ok": false, "error": null, "completion_tokens": 454, "reasoning_chars": 1538, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-296", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-535", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-172", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-23", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-43", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-38", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-97", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-657", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-814", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-346", "ok": true, "error": null, "completion_tokens": 295, "reasoning_chars": 1062, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-365", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-384", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-677", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-486", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-55", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-680", "ok": true, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-510", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-128", "ok": true, "error": null, "completion_tokens": 174, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-805", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-394", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-435", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-285", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-687", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 864, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-588", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-100", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-399", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-266", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-693", "ok": true, "error": null, "completion_tokens": 167, "reasoning_chars": 476, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-532", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-558", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-319", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-459", "ok": true, "error": null, "completion_tokens": 340, "reasoning_chars": 1109, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-675", "ok": false, "error": null, "completion_tokens": 440, "reasoning_chars": 1887, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-129", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-331", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-12", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-164", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-306", "ok": true, "error": null, "completion_tokens": 65, "reasoning_chars": 272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-275", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-418", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-264", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-94", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-389", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-427", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-269", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-643", "ok": false, "error": null, "completion_tokens": 253, "reasoning_chars": 1062, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-7", "ok": true, "error": null, "completion_tokens": 808, "reasoning_chars": 2672, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-419", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-616", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-699", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-473", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-523", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-214", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-376", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-123", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-200", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-493", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-474", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-387", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-746", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-790", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-181", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-413", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-349", "ok": false, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-434", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-601", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-489", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-286", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-488", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-759", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-273", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-59", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-315", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-425", "ok": true, "error": null, "completion_tokens": 405, "reasoning_chars": 782, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-58", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-691", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 1215, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-497", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-515", "ok": false, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-398", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-670", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-170", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-334", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-307", "ok": false, "error": null, "completion_tokens": 648, "reasoning_chars": 1961, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-495", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 853, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-597", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-775", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-186", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-733", "ok": false, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-308", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-5", "ok": false, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-231", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 823, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "tqa-356", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-810", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-270", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-13", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-114", "ok": true, "error": null, "completion_tokens": 78, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-4", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-417", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "tqa-750", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "gsm8k": { "task": "gsm8k", "n": 100, "hits": 47, "accuracy_pct": 47.0, "accuracy_ex_trunc_pct": 47.0, "truncated": 0, "completion_tokens_p50": 8, "completion_tokens_p90": 198, "completion_tokens_max": 2326, "reasoning_chars_p50": 0, "reasoning_chars_p90": 408, "reasoning_leaked": null, "empty": 0, "length_limited": 0, "errors": 0, "elapsed_s": 8003.5, "gen_tokens": 7134, "tok_per_s": 0.9, "details": [ { "id": "gsm8k-144", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-627", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 698, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-195", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-201", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-730", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-166", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1141", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-832", "ok": true, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-53", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1042", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-609", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-318", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-892", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-204", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1120", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1040", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 436, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-556", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1015", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-206", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 623, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1058", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1044", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-131", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1154", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-158", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1149", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1156", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-60", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-918", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1010", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1032", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1053", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-577", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-935", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-692", "ok": true, "error": null, "completion_tokens": 226, "reasoning_chars": 442, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-153", "ok": true, "error": null, "completion_tokens": 516, "reasoning_chars": 1504, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-40", "ok": true, "error": null, "completion_tokens": 254, "reasoning_chars": 705, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-613", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-875", "ok": true, "error": null, "completion_tokens": 49, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-189", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-800", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-169", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1169", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-211", "ok": true, "error": null, "completion_tokens": 166, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1183", "ok": true, "error": null, "completion_tokens": 2326, "reasoning_chars": 9080, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-193", "ok": true, "error": null, "completion_tokens": 210, "reasoning_chars": 777, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-230", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-787", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-83", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1318", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-733", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1205", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-88", "ok": false, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-509", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-311", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-874", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-287", "ok": true, "error": null, "completion_tokens": 139, "reasoning_chars": 351, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1055", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1246", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-975", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-280", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-541", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-974", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-827", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-125", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-453", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-942", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1199", "ok": false, "error": null, "completion_tokens": 36, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-884", "ok": true, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-855", "ok": true, "error": null, "completion_tokens": 135, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1270", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-349", "ok": false, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1231", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1106", "ok": true, "error": null, "completion_tokens": 33, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-803", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-922", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-152", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-920", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-207", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-297", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-786", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-101", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1301", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-27", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-1005", "ok": true, "error": null, "completion_tokens": 462, "reasoning_chars": 1220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-659", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-340", "ok": true, "error": null, "completion_tokens": 592, "reasoning_chars": 2097, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-288", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-763", "ok": true, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-840", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-742", "ok": true, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-222", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-468", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-220", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-662", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-269", "ok": true, "error": null, "completion_tokens": 87, "reasoning_chars": 325, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "gsm8k-1193", "ok": false, "error": null, "completion_tokens": 12, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-270", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-114", "ok": true, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-728", "ok": false, "error": null, "completion_tokens": 6, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "gsm8k-396", "ok": true, "error": null, "completion_tokens": 198, "reasoning_chars": 408, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "math500": { "task": "math500", "n": 100, "hits": 67, "accuracy_pct": 67.0, "accuracy_ex_trunc_pct": 67.68, "truncated": 1, "completion_tokens_p50": 356, "completion_tokens_p90": 3016, "completion_tokens_max": 32768, "reasoning_chars_p50": 575, "reasoning_chars_p90": 5961, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 10767.6, "gen_tokens": 135071, "tok_per_s": 12.5, "details": [ { "id": "math-test/algebra/1553.json", "ok": true, "error": null, "completion_tokens": 475, "reasoning_chars": 664, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/538.json", "ok": true, "error": null, "completion_tokens": 521, "reasoning_chars": 886, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/691.json", "ok": true, "error": null, "completion_tokens": 15, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1090.json", "ok": true, "error": null, "completion_tokens": 758, "reasoning_chars": 1603, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/116.json", "ok": true, "error": null, "completion_tokens": 201, "reasoning_chars": 282, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1255.json", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 189, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1143.json", "ok": true, "error": null, "completion_tokens": 728, "reasoning_chars": 908, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1201.json", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 40262, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "math-test/intermediate_algebra/964.json", "ok": true, "error": null, "completion_tokens": 3267, "reasoning_chars": 5961, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/662.json", "ok": false, "error": null, "completion_tokens": 2763, "reasoning_chars": 4403, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/153.json", "ok": false, "error": null, "completion_tokens": 219, "reasoning_chars": 350, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/276.json", "ok": true, "error": null, "completion_tokens": 390, "reasoning_chars": 545, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1840.json", "ok": false, "error": null, "completion_tokens": 11, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1072.json", "ok": false, "error": null, "completion_tokens": 437, "reasoning_chars": 609, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1907.json", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/567.json", "ok": true, "error": null, "completion_tokens": 377, "reasoning_chars": 552, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1922.json", "ok": true, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1779.json", "ok": true, "error": null, "completion_tokens": 4947, "reasoning_chars": 11548, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/951.json", "ok": true, "error": null, "completion_tokens": 239, "reasoning_chars": 318, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1197.json", "ok": false, "error": null, "completion_tokens": 5097, "reasoning_chars": 10481, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/853.json", "ok": true, "error": null, "completion_tokens": 882, "reasoning_chars": 1542, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/516.json", "ok": false, "error": null, "completion_tokens": 521, "reasoning_chars": 911, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1114.json", "ok": false, "error": null, "completion_tokens": 244, "reasoning_chars": 575, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1065.json", "ok": true, "error": null, "completion_tokens": 445, "reasoning_chars": 524, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1232.json", "ok": false, "error": null, "completion_tokens": 291, "reasoning_chars": 305, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/769.json", "ok": true, "error": null, "completion_tokens": 805, "reasoning_chars": 856, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1791.json", "ok": false, "error": null, "completion_tokens": 1869, "reasoning_chars": 3832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/246.json", "ok": true, "error": null, "completion_tokens": 308, "reasoning_chars": 484, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1166.json", "ok": true, "error": null, "completion_tokens": 4106, "reasoning_chars": 7824, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1410.json", "ok": false, "error": null, "completion_tokens": 420, "reasoning_chars": 705, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1686.json", "ok": true, "error": null, "completion_tokens": 217, "reasoning_chars": 422, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/754.json", "ok": false, "error": null, "completion_tokens": 963, "reasoning_chars": 1903, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/477.json", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 1334, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2680.json", "ok": true, "error": null, "completion_tokens": 151, "reasoning_chars": 464, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/23957.json", "ok": true, "error": null, "completion_tokens": 149, "reasoning_chars": 266, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1865.json", "ok": true, "error": null, "completion_tokens": 679, "reasoning_chars": 1618, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1014.json", "ok": true, "error": null, "completion_tokens": 356, "reasoning_chars": 745, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/431.json", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 246, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1425.json", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 232, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2584.json", "ok": false, "error": null, "completion_tokens": 228, "reasoning_chars": 309, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2470.json", "ok": true, "error": null, "completion_tokens": 147, "reasoning_chars": 50, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/547.json", "ok": true, "error": null, "completion_tokens": 3016, "reasoning_chars": 8666, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1105.json", "ok": true, "error": null, "completion_tokens": 31, "reasoning_chars": 23, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/347.json", "ok": true, "error": null, "completion_tokens": 243, "reasoning_chars": 634, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1257.json", "ok": true, "error": null, "completion_tokens": 845, "reasoning_chars": 1576, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1126.json", "ok": true, "error": null, "completion_tokens": 573, "reasoning_chars": 993, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/239.json", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1961.json", "ok": false, "error": null, "completion_tokens": 270, "reasoning_chars": 631, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1388.json", "ok": true, "error": null, "completion_tokens": 197, "reasoning_chars": 231, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1078.json", "ok": true, "error": null, "completion_tokens": 577, "reasoning_chars": 1097, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2157.json", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/339.json", "ok": true, "error": null, "completion_tokens": 361, "reasoning_chars": 1295, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1761.json", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1960.json", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 238, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2086.json", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1462.json", "ok": true, "error": null, "completion_tokens": 2757, "reasoning_chars": 4691, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/1060.json", "ok": true, "error": null, "completion_tokens": 127, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/1055.json", "ok": true, "error": null, "completion_tokens": 2301, "reasoning_chars": 6425, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2517.json", "ok": false, "error": null, "completion_tokens": 519, "reasoning_chars": 1019, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/996.json", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/893.json", "ok": true, "error": null, "completion_tokens": 431, "reasoning_chars": 507, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/51.json", "ok": false, "error": null, "completion_tokens": 330, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1291.json", "ok": true, "error": null, "completion_tokens": 387, "reasoning_chars": 808, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/694.json", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/711.json", "ok": false, "error": null, "completion_tokens": 8904, "reasoning_chars": 29317, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1655.json", "ok": false, "error": null, "completion_tokens": 251, "reasoning_chars": 391, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1936.json", "ok": true, "error": null, "completion_tokens": 41, "reasoning_chars": 49, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1572.json", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 747, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/509.json", "ok": false, "error": null, "completion_tokens": 672, "reasoning_chars": 968, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/864.json", "ok": true, "error": null, "completion_tokens": 73, "reasoning_chars": 182, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1405.json", "ok": true, "error": null, "completion_tokens": 1368, "reasoning_chars": 2486, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/192.json", "ok": true, "error": null, "completion_tokens": 128, "reasoning_chars": 190, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2102.json", "ok": true, "error": null, "completion_tokens": 345, "reasoning_chars": 500, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/843.json", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/187.json", "ok": true, "error": null, "completion_tokens": 182, "reasoning_chars": 432, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/612.json", "ok": true, "error": null, "completion_tokens": 443, "reasoning_chars": 1629, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/615.json", "ok": true, "error": null, "completion_tokens": 1944, "reasoning_chars": 5655, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1908.json", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/928.json", "ok": true, "error": null, "completion_tokens": 971, "reasoning_chars": 2150, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/505.json", "ok": true, "error": null, "completion_tokens": 44, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1930.json", "ok": false, "error": null, "completion_tokens": 2893, "reasoning_chars": 5553, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1934.json", "ok": true, "error": null, "completion_tokens": 138, "reasoning_chars": 201, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1139.json", "ok": true, "error": null, "completion_tokens": 13208, "reasoning_chars": 37605, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1000.json", "ok": true, "error": null, "completion_tokens": 3736, "reasoning_chars": 7900, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/183.json", "ok": true, "error": null, "completion_tokens": 750, "reasoning_chars": 1277, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/1275.json", "ok": true, "error": null, "completion_tokens": 566, "reasoning_chars": 1166, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2700.json", "ok": false, "error": null, "completion_tokens": 511, "reasoning_chars": 1306, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2257.json", "ok": false, "error": null, "completion_tokens": 179, "reasoning_chars": 399, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/14.json", "ok": false, "error": null, "completion_tokens": 245, "reasoning_chars": 686, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/1300.json", "ok": false, "error": null, "completion_tokens": 702, "reasoning_chars": 828, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/number_theory/515.json", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 473, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1423.json", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/intermediate_algebra/1102.json", "ok": true, "error": null, "completion_tokens": 888, "reasoning_chars": 1896, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1646.json", "ok": true, "error": null, "completion_tokens": 10531, "reasoning_chars": 30800, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/1272.json", "ok": false, "error": null, "completion_tokens": 130, "reasoning_chars": 171, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/counting_and_probability/25149.json", "ok": true, "error": null, "completion_tokens": 286, "reasoning_chars": 440, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/precalculus/697.json", "ok": false, "error": null, "completion_tokens": 3048, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/algebra/2036.json", "ok": true, "error": null, "completion_tokens": 122, "reasoning_chars": 277, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "math-test/prealgebra/2037.json", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "math-test/geometry/353.json", "ok": true, "error": null, "completion_tokens": 290, "reasoning_chars": 564, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "bbh": { "task": "bbh", "n": 150, "hits": 82, "accuracy_pct": 54.67, "accuracy_ex_trunc_pct": 54.36, "truncated": 1, "completion_tokens_p50": 53, "completion_tokens_p90": 663, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 2348, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 8270.4, "gen_tokens": 96814, "tok_per_s": 11.7, "details": [ { "id": "bbh-sports_understanding-101", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-60", "ok": true, "error": null, "completion_tokens": 527, "reasoning_chars": 1855, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-113", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-31", "ok": true, "error": null, "completion_tokens": 1570, "reasoning_chars": 5969, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-144", "ok": true, "error": null, "completion_tokens": 663, "reasoning_chars": 1853, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-28", "ok": true, "error": null, "completion_tokens": 370, "reasoning_chars": 464, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-116", "ok": true, "error": null, "completion_tokens": 82, "reasoning_chars": 148, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-67", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-3", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 765, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-29", "ok": false, "error": null, "completion_tokens": 56, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-192", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 1206, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-196", "ok": true, "error": null, "completion_tokens": 545, "reasoning_chars": 2162, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-10", "ok": false, "error": null, "completion_tokens": 47, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-83", "ok": true, "error": null, "completion_tokens": 165, "reasoning_chars": 728, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-76", "ok": false, "error": null, "completion_tokens": 52, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-212", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-87", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-1", "ok": true, "error": null, "completion_tokens": 1206, "reasoning_chars": 4589, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-140", "ok": true, "error": null, "completion_tokens": 415, "reasoning_chars": 1814, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-8", "ok": true, "error": null, "completion_tokens": 392, "reasoning_chars": 1324, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-138", "ok": true, "error": null, "completion_tokens": 273, "reasoning_chars": 1134, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-221", "ok": true, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-136", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-14", "ok": true, "error": null, "completion_tokens": 38, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-0", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-207", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-32", "ok": true, "error": null, "completion_tokens": 573, "reasoning_chars": 2462, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-83", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 579, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-220", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-234", "ok": true, "error": null, "completion_tokens": 637, "reasoning_chars": 1718, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-61", "ok": true, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-21", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-31", "ok": true, "error": null, "completion_tokens": 189, "reasoning_chars": 346, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-13", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-211", "ok": true, "error": null, "completion_tokens": 949, "reasoning_chars": 3730, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-37", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-225", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-90", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 50, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-101", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-179", "ok": true, "error": null, "completion_tokens": 317, "reasoning_chars": 1056, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-173", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-105", "ok": true, "error": null, "completion_tokens": 164, "reasoning_chars": 612, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-88", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-68", "ok": true, "error": null, "completion_tokens": 241, "reasoning_chars": 796, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-185", "ok": true, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-227", "ok": true, "error": null, "completion_tokens": 134, "reasoning_chars": 391, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-129", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-238", "ok": true, "error": null, "completion_tokens": 272, "reasoning_chars": 674, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-202", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-93", "ok": true, "error": null, "completion_tokens": 279, "reasoning_chars": 1126, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-93", "ok": true, "error": null, "completion_tokens": 1197, "reasoning_chars": 4710, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-161", "ok": false, "error": null, "completion_tokens": 26, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-46", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-227", "ok": true, "error": null, "completion_tokens": 467, "reasoning_chars": 1578, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-158", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-205", "ok": true, "error": null, "completion_tokens": 981, "reasoning_chars": 2483, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-89", "ok": true, "error": null, "completion_tokens": 187, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-177", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-65", "ok": true, "error": null, "completion_tokens": 256, "reasoning_chars": 682, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-72", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-49", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 410, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-54", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-64", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-86", "ok": false, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-108", "ok": true, "error": null, "completion_tokens": 89, "reasoning_chars": 358, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-194", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-194", "ok": true, "error": null, "completion_tokens": 150, "reasoning_chars": 647, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-75", "ok": true, "error": null, "completion_tokens": 32768, "reasoning_chars": 69171, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "bbh-tracking_shuffled_objects_three_objects-106", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-81", "ok": true, "error": null, "completion_tokens": 208, "reasoning_chars": 864, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-242", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-150", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-25", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-139", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-12", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 1042, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-178", "ok": true, "error": null, "completion_tokens": 258, "reasoning_chars": 944, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-160", "ok": false, "error": null, "completion_tokens": 27, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-115", "ok": true, "error": null, "completion_tokens": 11269, "reasoning_chars": 45590, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_five_objects-44", "ok": true, "error": null, "completion_tokens": 997, "reasoning_chars": 3708, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-189", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 1307, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-102", "ok": true, "error": null, "completion_tokens": 305, "reasoning_chars": 1081, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-38", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-141", "ok": false, "error": null, "completion_tokens": 37, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-38", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-168", "ok": true, "error": null, "completion_tokens": 598, "reasoning_chars": 1396, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-212", "ok": false, "error": null, "completion_tokens": 18650, "reasoning_chars": 49380, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-245", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-222", "ok": true, "error": null, "completion_tokens": 292, "reasoning_chars": 1083, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-244", "ok": true, "error": null, "completion_tokens": 318, "reasoning_chars": 1159, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-35", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-126", "ok": false, "error": null, "completion_tokens": 56, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-90", "ok": false, "error": null, "completion_tokens": 53, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-5", "ok": true, "error": null, "completion_tokens": 238, "reasoning_chars": 970, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-241", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-234", "ok": true, "error": null, "completion_tokens": 562, "reasoning_chars": 1388, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-233", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-5", "ok": false, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-152", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-9", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-209", "ok": false, "error": null, "completion_tokens": 5, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-171", "ok": true, "error": null, "completion_tokens": 540, "reasoning_chars": 1790, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-navigate-160", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-166", "ok": false, "error": null, "completion_tokens": 159, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-213", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-reasoning_about_colored_objects-240", "ok": true, "error": null, "completion_tokens": 45, "reasoning_chars": 77, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-95", "ok": true, "error": null, "completion_tokens": 126, "reasoning_chars": 479, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_five_objects-227", "ok": true, "error": null, "completion_tokens": 511, "reasoning_chars": 1513, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-96", "ok": true, "error": null, "completion_tokens": 897, "reasoning_chars": 3880, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-33", "ok": true, "error": null, "completion_tokens": 566, "reasoning_chars": 102, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-4", "ok": true, "error": null, "completion_tokens": 347, "reasoning_chars": 1154, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-180", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-74", "ok": true, "error": null, "completion_tokens": 304, "reasoning_chars": 1217, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-88", "ok": false, "error": null, "completion_tokens": 10, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-10", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-191", "ok": true, "error": null, "completion_tokens": 1805, "reasoning_chars": 6968, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-71", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-disambiguation_qa-150", "ok": false, "error": null, "completion_tokens": 22, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-188", "ok": true, "error": null, "completion_tokens": 255, "reasoning_chars": 317, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-191", "ok": true, "error": null, "completion_tokens": 236, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-web_of_lies-203", "ok": true, "error": null, "completion_tokens": 186, "reasoning_chars": 695, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-237", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 584, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-penguins_in_a_table-24", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-27", "ok": true, "error": null, "completion_tokens": 512, "reasoning_chars": 1351, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-177", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-boolean_expressions-192", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-242", "ok": true, "error": null, "completion_tokens": 191, "reasoning_chars": 453, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-dyck_languages-51", "ok": true, "error": null, "completion_tokens": 2429, "reasoning_chars": 9954, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_three_objects-96", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-word_sorting-100", "ok": false, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-204", "ok": false, "error": null, "completion_tokens": 7, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_seven_objects-58", "ok": true, "error": null, "completion_tokens": 610, "reasoning_chars": 1893, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-object_counting-157", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-104", "ok": true, "error": null, "completion_tokens": 35, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-47", "ok": false, "error": null, "completion_tokens": 24, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-date_understanding-150", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-multistep_arithmetic_two-23", "ok": true, "error": null, "completion_tokens": 1116, "reasoning_chars": 2348, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-sports_understanding-68", "ok": false, "error": null, "completion_tokens": 8, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-174", "ok": true, "error": null, "completion_tokens": 206, "reasoning_chars": 306, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-tracking_shuffled_objects_seven_objects-108", "ok": true, "error": null, "completion_tokens": 771, "reasoning_chars": 2620, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-203", "ok": false, "error": null, "completion_tokens": 342, "reasoning_chars": 615, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-formal_fallacies-115", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-hyperbaton-223", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-salient_translation_error_detection-221", "ok": true, "error": null, "completion_tokens": 265, "reasoning_chars": 352, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-temporal_sequences-5", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-geometric_shapes-233", "ok": true, "error": null, "completion_tokens": 253, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-ruin_names-3", "ok": true, "error": null, "completion_tokens": 196, "reasoning_chars": 708, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-logical_deduction_three_objects-119", "ok": true, "error": null, "completion_tokens": 184, "reasoning_chars": 588, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-movie_recommendation-197", "ok": false, "error": null, "completion_tokens": 9, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "bbh-snarks-103", "ok": true, "error": null, "completion_tokens": 940, "reasoning_chars": 3978, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "bbh-causal_judgement-178", "ok": false, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false } ] }, "humaneval": { "task": "humaneval", "n": 164, "hits": 148, "accuracy_pct": 90.24, "accuracy_ex_trunc_pct": 91.36, "truncated": 2, "completion_tokens_p50": 477, "completion_tokens_p90": 1716, "completion_tokens_max": 32768, "reasoning_chars_p50": 1219, "reasoning_chars_p90": 5765, "reasoning_leaked": null, "empty": 2, "length_limited": 2, "errors": 0, "elapsed_s": 11691.9, "gen_tokens": 244101, "tok_per_s": 20.9, "details": [ { "id": "HumanEval/0", "ok": true, "error": null, "completion_tokens": 341, "reasoning_chars": 1253, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/1", "ok": false, "error": null, "completion_tokens": 998, "reasoning_chars": 4041, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/2", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 924, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/3", "ok": true, "error": null, "completion_tokens": 247, "reasoning_chars": 899, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/4", "ok": true, "error": null, "completion_tokens": 480, "reasoning_chars": 1839, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/5", "ok": true, "error": null, "completion_tokens": 559, "reasoning_chars": 2337, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/6", "ok": true, "error": null, "completion_tokens": 409, "reasoning_chars": 1371, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/7", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 1676, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/8", "ok": true, "error": null, "completion_tokens": 219, "reasoning_chars": 695, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/9", "ok": true, "error": null, "completion_tokens": 778, "reasoning_chars": 2979, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/10", "ok": true, "error": null, "completion_tokens": 1524, "reasoning_chars": 5765, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/11", "ok": true, "error": null, "completion_tokens": 2393, "reasoning_chars": 8836, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/12", "ok": false, "error": null, "completion_tokens": 31, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/13", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/14", "ok": true, "error": null, "completion_tokens": 215, "reasoning_chars": 626, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/15", "ok": true, "error": null, "completion_tokens": 320, "reasoning_chars": 915, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/16", "ok": true, "error": null, "completion_tokens": 21, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/17", "ok": true, "error": null, "completion_tokens": 418, "reasoning_chars": 804, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/18", "ok": true, "error": null, "completion_tokens": 564, "reasoning_chars": 1664, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/19", "ok": true, "error": null, "completion_tokens": 461, "reasoning_chars": 1261, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/20", "ok": true, "error": null, "completion_tokens": 1585, "reasoning_chars": 6930, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/21", "ok": true, "error": null, "completion_tokens": 1729, "reasoning_chars": 5940, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/22", "ok": false, "error": null, "completion_tokens": 1950, "reasoning_chars": 7568, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/23", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 559, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/24", "ok": true, "error": null, "completion_tokens": 542, "reasoning_chars": 1608, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/25", "ok": true, "error": null, "completion_tokens": 1035, "reasoning_chars": 2803, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/26", "ok": false, "error": null, "completion_tokens": 813, "reasoning_chars": 3512, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/27", "ok": true, "error": null, "completion_tokens": 416, "reasoning_chars": 1443, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/28", "ok": true, "error": null, "completion_tokens": 306, "reasoning_chars": 1114, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/29", "ok": true, "error": null, "completion_tokens": 213, "reasoning_chars": 581, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/30", "ok": true, "error": null, "completion_tokens": 229, "reasoning_chars": 434, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/31", "ok": true, "error": null, "completion_tokens": 462, "reasoning_chars": 1007, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/32", "ok": false, "error": null, "completion_tokens": 25587, "reasoning_chars": 81942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/33", "ok": true, "error": null, "completion_tokens": 489, "reasoning_chars": 895, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/34", "ok": true, "error": null, "completion_tokens": 171, "reasoning_chars": 425, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/35", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/36", "ok": true, "error": null, "completion_tokens": 753, "reasoning_chars": 1783, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/37", "ok": true, "error": null, "completion_tokens": 781, "reasoning_chars": 2292, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/38", "ok": false, "error": null, "completion_tokens": 115, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/39", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 41897, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/40", "ok": true, "error": null, "completion_tokens": 461, "reasoning_chars": 873, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/41", "ok": true, "error": null, "completion_tokens": 555, "reasoning_chars": 1785, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/42", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 439, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/43", "ok": true, "error": null, "completion_tokens": 373, "reasoning_chars": 772, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/44", "ok": true, "error": null, "completion_tokens": 421, "reasoning_chars": 948, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/45", "ok": true, "error": null, "completion_tokens": 17, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/46", "ok": true, "error": null, "completion_tokens": 920, "reasoning_chars": 1681, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/47", "ok": true, "error": null, "completion_tokens": 518, "reasoning_chars": 1480, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/48", "ok": true, "error": null, "completion_tokens": 140, "reasoning_chars": 262, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/49", "ok": true, "error": null, "completion_tokens": 23, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/50", "ok": false, "error": null, "completion_tokens": 42, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/51", "ok": true, "error": null, "completion_tokens": 294, "reasoning_chars": 575, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/52", "ok": true, "error": null, "completion_tokens": 34, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/53", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/54", "ok": true, "error": null, "completion_tokens": 381, "reasoning_chars": 804, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/55", "ok": true, "error": null, "completion_tokens": 411, "reasoning_chars": 942, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/56", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 734, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/57", "ok": true, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/58", "ok": true, "error": null, "completion_tokens": 407, "reasoning_chars": 1131, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/59", "ok": true, "error": null, "completion_tokens": 552, "reasoning_chars": 1382, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/60", "ok": true, "error": null, "completion_tokens": 192, "reasoning_chars": 304, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/61", "ok": true, "error": null, "completion_tokens": 285, "reasoning_chars": 612, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/62", "ok": true, "error": null, "completion_tokens": 223, "reasoning_chars": 442, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/63", "ok": true, "error": null, "completion_tokens": 453, "reasoning_chars": 721, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/64", "ok": true, "error": null, "completion_tokens": 331, "reasoning_chars": 626, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/65", "ok": true, "error": null, "completion_tokens": 2378, "reasoning_chars": 8115, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/66", "ok": true, "error": null, "completion_tokens": 275, "reasoning_chars": 530, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/67", "ok": true, "error": null, "completion_tokens": 293, "reasoning_chars": 1094, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/68", "ok": true, "error": null, "completion_tokens": 713, "reasoning_chars": 2394, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/69", "ok": true, "error": null, "completion_tokens": 477, "reasoning_chars": 1522, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/70", "ok": true, "error": null, "completion_tokens": 773, "reasoning_chars": 1989, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/71", "ok": true, "error": null, "completion_tokens": 692, "reasoning_chars": 1707, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/72", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 355, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/73", "ok": true, "error": null, "completion_tokens": 551, "reasoning_chars": 1278, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/74", "ok": true, "error": null, "completion_tokens": 371, "reasoning_chars": 546, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/75", "ok": true, "error": null, "completion_tokens": 1028, "reasoning_chars": 2896, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/76", "ok": true, "error": null, "completion_tokens": 4313, "reasoning_chars": 11739, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/77", "ok": true, "error": null, "completion_tokens": 891, "reasoning_chars": 2223, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/78", "ok": true, "error": null, "completion_tokens": 568, "reasoning_chars": 713, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/79", "ok": true, "error": null, "completion_tokens": 348, "reasoning_chars": 727, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/80", "ok": true, "error": null, "completion_tokens": 394, "reasoning_chars": 705, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/81", "ok": true, "error": null, "completion_tokens": 1000, "reasoning_chars": 2028, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/82", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 786, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/83", "ok": true, "error": null, "completion_tokens": 866, "reasoning_chars": 2290, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/84", "ok": true, "error": null, "completion_tokens": 244, "reasoning_chars": 241, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/85", "ok": true, "error": null, "completion_tokens": 250, "reasoning_chars": 625, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/86", "ok": false, "error": null, "completion_tokens": 169, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/87", "ok": true, "error": null, "completion_tokens": 386, "reasoning_chars": 1314, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/88", "ok": true, "error": null, "completion_tokens": 565, "reasoning_chars": 1158, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/89", "ok": true, "error": null, "completion_tokens": 548, "reasoning_chars": 1286, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/90", "ok": true, "error": null, "completion_tokens": 423, "reasoning_chars": 1133, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/91", "ok": true, "error": null, "completion_tokens": 5826, "reasoning_chars": 21272, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/92", "ok": true, "error": null, "completion_tokens": 528, "reasoning_chars": 1831, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/93", "ok": true, "error": null, "completion_tokens": 1387, "reasoning_chars": 4220, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/94", "ok": true, "error": null, "completion_tokens": 1097, "reasoning_chars": 2695, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/95", "ok": true, "error": null, "completion_tokens": 616, "reasoning_chars": 1566, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/96", "ok": true, "error": null, "completion_tokens": 1258, "reasoning_chars": 3316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/97", "ok": true, "error": null, "completion_tokens": 286, "reasoning_chars": 520, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/98", "ok": true, "error": null, "completion_tokens": 299, "reasoning_chars": 594, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/99", "ok": true, "error": null, "completion_tokens": 23382, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/100", "ok": true, "error": null, "completion_tokens": 667, "reasoning_chars": 1755, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/101", "ok": true, "error": null, "completion_tokens": 352, "reasoning_chars": 944, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/102", "ok": false, "error": null, "completion_tokens": 46, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/103", "ok": true, "error": null, "completion_tokens": 5435, "reasoning_chars": 17998, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/104", "ok": true, "error": null, "completion_tokens": 493, "reasoning_chars": 1290, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/105", "ok": true, "error": null, "completion_tokens": 1267, "reasoning_chars": 4350, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/106", "ok": true, "error": null, "completion_tokens": 849, "reasoning_chars": 2039, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/107", "ok": true, "error": null, "completion_tokens": 537, "reasoning_chars": 728, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/108", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 35297, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true }, { "id": "HumanEval/109", "ok": true, "error": null, "completion_tokens": 608, "reasoning_chars": 1832, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/110", "ok": true, "error": null, "completion_tokens": 549, "reasoning_chars": 1111, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/111", "ok": true, "error": null, "completion_tokens": 712, "reasoning_chars": 1971, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/112", "ok": true, "error": null, "completion_tokens": 376, "reasoning_chars": 812, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/113", "ok": true, "error": null, "completion_tokens": 2496, "reasoning_chars": 7733, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/114", "ok": true, "error": null, "completion_tokens": 72, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/115", "ok": true, "error": null, "completion_tokens": 523, "reasoning_chars": 664, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/116", "ok": true, "error": null, "completion_tokens": 83, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/117", "ok": true, "error": null, "completion_tokens": 732, "reasoning_chars": 1843, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/118", "ok": true, "error": null, "completion_tokens": 1017, "reasoning_chars": 2891, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/119", "ok": true, "error": null, "completion_tokens": 727, "reasoning_chars": 1783, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/120", "ok": true, "error": null, "completion_tokens": 369, "reasoning_chars": 1274, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/121", "ok": true, "error": null, "completion_tokens": 323, "reasoning_chars": 565, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/122", "ok": true, "error": null, "completion_tokens": 48, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/123", "ok": true, "error": null, "completion_tokens": 1015, "reasoning_chars": 2534, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/124", "ok": true, "error": null, "completion_tokens": 237, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/125", "ok": true, "error": null, "completion_tokens": 1240, "reasoning_chars": 3874, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/126", "ok": true, "error": null, "completion_tokens": 651, "reasoning_chars": 2188, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/127", "ok": false, "error": null, "completion_tokens": 706, "reasoning_chars": 2100, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/128", "ok": true, "error": null, "completion_tokens": 469, "reasoning_chars": 977, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/129", "ok": true, "error": null, "completion_tokens": 1916, "reasoning_chars": 6406, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/130", "ok": false, "error": null, "completion_tokens": 1018, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/131", "ok": true, "error": null, "completion_tokens": 379, "reasoning_chars": 931, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/132", "ok": false, "error": null, "completion_tokens": 3759, "reasoning_chars": 14637, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/133", "ok": true, "error": null, "completion_tokens": 25, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/134", "ok": true, "error": null, "completion_tokens": 1058, "reasoning_chars": 3744, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/135", "ok": true, "error": null, "completion_tokens": 511, "reasoning_chars": 1384, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/136", "ok": true, "error": null, "completion_tokens": 463, "reasoning_chars": 977, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/137", "ok": true, "error": null, "completion_tokens": 825, "reasoning_chars": 2511, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/138", "ok": true, "error": null, "completion_tokens": 251, "reasoning_chars": 565, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/139", "ok": true, "error": null, "completion_tokens": 708, "reasoning_chars": 1977, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/140", "ok": true, "error": null, "completion_tokens": 732, "reasoning_chars": 2493, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/141", "ok": true, "error": null, "completion_tokens": 112, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/142", "ok": true, "error": null, "completion_tokens": 997, "reasoning_chars": 2330, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/143", "ok": true, "error": null, "completion_tokens": 568, "reasoning_chars": 1219, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/144", "ok": true, "error": null, "completion_tokens": 495, "reasoning_chars": 1582, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/145", "ok": false, "error": null, "completion_tokens": 17983, "reasoning_chars": 55975, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/146", "ok": true, "error": null, "completion_tokens": 1716, "reasoning_chars": 4524, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/147", "ok": true, "error": null, "completion_tokens": 3878, "reasoning_chars": 9333, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/148", "ok": true, "error": null, "completion_tokens": 468, "reasoning_chars": 627, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/149", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 974, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/150", "ok": true, "error": null, "completion_tokens": 432, "reasoning_chars": 957, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/151", "ok": true, "error": null, "completion_tokens": 1495, "reasoning_chars": 4761, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/152", "ok": true, "error": null, "completion_tokens": 339, "reasoning_chars": 400, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/153", "ok": true, "error": null, "completion_tokens": 734, "reasoning_chars": 1395, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/154", "ok": true, "error": null, "completion_tokens": 570, "reasoning_chars": 1914, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/155", "ok": true, "error": null, "completion_tokens": 525, "reasoning_chars": 1426, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/156", "ok": true, "error": null, "completion_tokens": 425, "reasoning_chars": 494, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/157", "ok": true, "error": null, "completion_tokens": 43, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/158", "ok": true, "error": null, "completion_tokens": 642, "reasoning_chars": 1959, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/159", "ok": true, "error": null, "completion_tokens": 453, "reasoning_chars": 528, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/160", "ok": false, "error": null, "completion_tokens": 112, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false }, { "id": "HumanEval/161", "ok": true, "error": null, "completion_tokens": 334, "reasoning_chars": 779, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/162", "ok": true, "error": null, "completion_tokens": 378, "reasoning_chars": 1234, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false }, { "id": "HumanEval/163", "ok": true, "error": null, "completion_tokens": 1527, "reasoning_chars": 4222, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false } ] }, "ifeval": { "task": "ifeval", "n": 100, "hits": 60, "accuracy_pct": 60.0, "accuracy_ex_trunc_pct": 60.61, "truncated": 1, "completion_tokens_p50": 305, "completion_tokens_p90": 2225, "completion_tokens_max": 32768, "reasoning_chars_p50": 0, "reasoning_chars_p90": 3925, "reasoning_leaked": null, "empty": 1, "length_limited": 1, "errors": 0, "elapsed_s": 8938.4, "gen_tokens": 140957, "tok_per_s": 15.8, "instruction_level_pct": 74.67, "instructions_checked": 150, "details": [ { "id": "ifeval-153", "ok": false, "error": null, "completion_tokens": 305, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 0, "skipped": false }, { "id": "ifeval-258", "ok": true, "error": null, "completion_tokens": 62, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-368", "ok": true, "error": null, "completion_tokens": 51, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-85", "ok": true, "error": null, "completion_tokens": 1083, "reasoning_chars": 3588, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-246", "ok": true, "error": null, "completion_tokens": 313, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-23", "ok": true, "error": null, "completion_tokens": 2, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-372", "ok": true, "error": null, "completion_tokens": 1543, "reasoning_chars": 3925, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-387", "ok": true, "error": null, "completion_tokens": 844, "reasoning_chars": 3164, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-138", "ok": true, "error": null, "completion_tokens": 407, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-56", "ok": true, "error": null, "completion_tokens": 266, "reasoning_chars": 425, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-277", "ok": false, "error": null, "completion_tokens": 2225, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-407", "ok": false, "error": null, "completion_tokens": 371, "reasoning_chars": 191, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-171", "ok": true, "error": null, "completion_tokens": 67, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-96", "ok": true, "error": null, "completion_tokens": 18199, "reasoning_chars": 22505, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-134", "ok": false, "error": null, "completion_tokens": 504, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-218", "ok": true, "error": null, "completion_tokens": 1257, "reasoning_chars": 685, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-330", "ok": true, "error": null, "completion_tokens": 39, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-6", "ok": false, "error": null, "completion_tokens": 101, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-143", "ok": true, "error": null, "completion_tokens": 5379, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-121", "ok": true, "error": null, "completion_tokens": 1412, "reasoning_chars": 4429, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-219", "ok": true, "error": null, "completion_tokens": 125, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-524", "ok": true, "error": null, "completion_tokens": 886, "reasoning_chars": 2181, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-206", "ok": true, "error": null, "completion_tokens": 195, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-144", "ok": true, "error": null, "completion_tokens": 446, "reasoning_chars": 1057, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-447", "ok": false, "error": null, "completion_tokens": 13, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-492", "ok": false, "error": null, "completion_tokens": 1142, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-485", "ok": false, "error": null, "completion_tokens": 549, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-220", "ok": true, "error": null, "completion_tokens": 1074, "reasoning_chars": 4316, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-412", "ok": false, "error": null, "completion_tokens": 479, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-342", "ok": false, "error": null, "completion_tokens": 738, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-222", "ok": true, "error": null, "completion_tokens": 7003, "reasoning_chars": 17411, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-131", "ok": false, "error": null, "completion_tokens": 287, "reasoning_chars": 834, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-210", "ok": false, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-137", "ok": true, "error": null, "completion_tokens": 193, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-464", "ok": true, "error": null, "completion_tokens": 64, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-503", "ok": true, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-446", "ok": true, "error": null, "completion_tokens": 322, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-2", "ok": true, "error": null, "completion_tokens": 194, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-525", "ok": false, "error": null, "completion_tokens": 66, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-27", "ok": false, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-136", "ok": true, "error": null, "completion_tokens": 505, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-286", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-449", "ok": true, "error": null, "completion_tokens": 1163, "reasoning_chars": 3970, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-55", "ok": false, "error": null, "completion_tokens": 3573, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-93", "ok": true, "error": null, "completion_tokens": 58, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-399", "ok": true, "error": null, "completion_tokens": 19, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-19", "ok": false, "error": null, "completion_tokens": 113, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-229", "ok": false, "error": null, "completion_tokens": 227, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-181", "ok": true, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-362", "ok": true, "error": null, "completion_tokens": 124, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-289", "ok": true, "error": null, "completion_tokens": 141, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-356", "ok": true, "error": null, "completion_tokens": 245, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-115", "ok": true, "error": null, "completion_tokens": 789, "reasoning_chars": 2928, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-404", "ok": false, "error": null, "completion_tokens": 547, "reasoning_chars": 2087, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-243", "ok": false, "error": null, "completion_tokens": 32768, "reasoning_chars": 96112, "empty": true, "finish_reason": "length", "content_mode": "unclosed-length-limit", "length_limited": true, "truncated": true, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-90", "ok": false, "error": null, "completion_tokens": 421, "reasoning_chars": 490, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-54", "ok": false, "error": null, "completion_tokens": 18, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-276", "ok": true, "error": null, "completion_tokens": 303, "reasoning_chars": 375, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-308", "ok": true, "error": null, "completion_tokens": 382, "reasoning_chars": 1056, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-64", "ok": false, "error": null, "completion_tokens": 931, "reasoning_chars": 3632, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-260", "ok": true, "error": null, "completion_tokens": 14, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-337", "ok": true, "error": null, "completion_tokens": 61, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-510", "ok": true, "error": null, "completion_tokens": 1079, "reasoning_chars": 3083, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-119", "ok": true, "error": null, "completion_tokens": 1329, "reasoning_chars": 4119, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-5", "ok": false, "error": null, "completion_tokens": 109, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-468", "ok": false, "error": null, "completion_tokens": 330, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-418", "ok": false, "error": null, "completion_tokens": 399, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-476", "ok": false, "error": null, "completion_tokens": 3108, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-49", "ok": true, "error": null, "completion_tokens": 4773, "reasoning_chars": 10209, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-336", "ok": true, "error": null, "completion_tokens": 307, "reasoning_chars": 788, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-422", "ok": false, "error": null, "completion_tokens": 305, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-417", "ok": true, "error": null, "completion_tokens": 107, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-347", "ok": false, "error": null, "completion_tokens": 16311, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-101", "ok": true, "error": null, "completion_tokens": 414, "reasoning_chars": 66, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-209", "ok": false, "error": null, "completion_tokens": 302, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-341", "ok": false, "error": null, "completion_tokens": 80, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 1, "skipped": false }, { "id": "ifeval-533", "ok": false, "error": null, "completion_tokens": 59, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 0, "skipped": false }, { "id": "ifeval-158", "ok": true, "error": null, "completion_tokens": 752, "reasoning_chars": 2232, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-321", "ok": true, "error": null, "completion_tokens": 802, "reasoning_chars": 3176, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-311", "ok": true, "error": null, "completion_tokens": 64, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-50", "ok": false, "error": null, "completion_tokens": 79, "reasoning_chars": 74, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 0, "passed": 0, "skipped": true }, { "id": "ifeval-518", "ok": true, "error": null, "completion_tokens": 1762, "reasoning_chars": 953, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-374", "ok": true, "error": null, "completion_tokens": 863, "reasoning_chars": 3305, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-317", "ok": true, "error": null, "completion_tokens": 1085, "reasoning_chars": 4231, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-512", "ok": true, "error": null, "completion_tokens": 57, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-102", "ok": false, "error": null, "completion_tokens": 4, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-72", "ok": false, "error": null, "completion_tokens": 96, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-33", "ok": false, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-263", "ok": true, "error": null, "completion_tokens": 168, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-501", "ok": true, "error": null, "completion_tokens": 8952, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-4", "ok": true, "error": null, "completion_tokens": 180, "reasoning_chars": 636, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-444", "ok": false, "error": null, "completion_tokens": 2150, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-228", "ok": true, "error": null, "completion_tokens": 93, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false }, { "id": "ifeval-299", "ok": true, "error": null, "completion_tokens": 3, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-278", "ok": false, "error": null, "completion_tokens": 16, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-154", "ok": true, "error": null, "completion_tokens": 2790, "reasoning_chars": 10978, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 3, "passed": 3, "skipped": false }, { "id": "ifeval-301", "ok": true, "error": null, "completion_tokens": 652, "reasoning_chars": 2115, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 2, "passed": 2, "skipped": false }, { "id": "ifeval-161", "ok": false, "error": null, "completion_tokens": 157, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 2, "passed": 1, "skipped": false }, { "id": "ifeval-394", "ok": false, "error": null, "completion_tokens": 129, "reasoning_chars": 0, "empty": false, "finish_reason": "stop", "content_mode": "publisher-direct-content", "length_limited": false, "truncated": false, "checked": 3, "passed": 2, "skipped": false }, { "id": "ifeval-159", "ok": true, "error": null, "completion_tokens": 115, "reasoning_chars": 414, "empty": false, "finish_reason": "stop", "content_mode": "explicit-close", "length_limited": false, "truncated": false, "checked": 1, "passed": 1, "skipped": false } ] } } } }