| { |
| "aime2026": { |
| "acc_len": 2.6565662068447096, |
| "acc_len_max": 4.318471337579618, |
| "acc_len_median": 2.6269313291099063, |
| "acc_len_min": 1.9606270943034945, |
| "concurrency": 1, |
| "elapsed_s": 3335.8, |
| "errors": 0, |
| "max_tokens": 16384, |
| "mean_completion_tokens": 6000, |
| "mean_prompt_tokens": 207, |
| "n_acc_matched": 30, |
| "n_acc_unmatched": 0, |
| "n_prompts": 30, |
| "n_prompts_expected": 30, |
| "n_requests": 30, |
| "output_tokens": 179990, |
| "output_tokens_per_s": 53.957, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.23665231526352995, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "aime2026", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "semantic_correct": 23, |
| "semantic_graded": 30, |
| "semantic_score": 0.7666666666666667, |
| "truncated": 6 |
| }, |
| "gsm8k": { |
| "acc_len": 5.851219913596805, |
| "acc_len_max": 8.0, |
| "acc_len_median": 5.913043478260869, |
| "acc_len_min": 2.851063829787234, |
| "concurrency": 1, |
| "elapsed_s": 1535.7, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 247, |
| "mean_prompt_tokens": 147, |
| "n_acc_matched": 1319, |
| "n_acc_unmatched": 0, |
| "n_prompts": 1319, |
| "n_prompts_expected": 1319, |
| "n_requests": 1319, |
| "output_tokens": 326026, |
| "output_tokens_per_s": 212.296, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.6930314162281149, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "gsm8k", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "semantic_correct": 976, |
| "semantic_graded": 1319, |
| "semantic_score": 0.7399545109931767, |
| "truncated": 0 |
| }, |
| "humaneval": { |
| "acc_len": 4.866431561653052, |
| "acc_len_max": 6.56989247311828, |
| "acc_len_median": 4.980739360049705, |
| "acc_len_min": 2.6048192771084335, |
| "concurrency": 1, |
| "elapsed_s": 810.0, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 999, |
| "mean_prompt_tokens": 220, |
| "n_acc_matched": 164, |
| "n_acc_unmatched": 0, |
| "n_prompts": 164, |
| "n_prompts_expected": 164, |
| "n_requests": 164, |
| "output_tokens": 163891, |
| "output_tokens_per_s": 202.325, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.552347365950436, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "humaneval", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "truncated": 1 |
| }, |
| "math500": { |
| "acc_len": 4.571222086925199, |
| "acc_len_max": 7.366666666666666, |
| "acc_len_median": 4.629166666666666, |
| "acc_len_min": 2.1540983606557376, |
| "concurrency": 1, |
| "elapsed_s": 3128.5, |
| "errors": 0, |
| "max_tokens": 8192, |
| "mean_completion_tokens": 791, |
| "mean_prompt_tokens": 156, |
| "n_acc_matched": 500, |
| "n_acc_unmatched": 0, |
| "n_prompts": 500, |
| "n_prompts_expected": 500, |
| "n_requests": 500, |
| "output_tokens": 395616, |
| "output_tokens_per_s": 126.456, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.510174583846457, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "math500", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "semantic_correct": 336, |
| "semantic_graded": 500, |
| "semantic_score": 0.672, |
| "truncated": 4 |
| }, |
| "mtbench": { |
| "acc_len": 3.4807646201963163, |
| "acc_len_max": 6.45945945945946, |
| "acc_len_median": 3.0978285971726898, |
| "acc_len_min": 2.0985915492957745, |
| "concurrency": 1, |
| "elapsed_s": 3010.8, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 1789, |
| "mean_prompt_tokens": 944, |
| "n_acc_matched": 160, |
| "n_acc_unmatched": 0, |
| "n_prompts": 80, |
| "n_prompts_expected": 80, |
| "n_requests": 160, |
| "output_tokens": 286305, |
| "output_tokens_per_s": 95.094, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.3543949457423309, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "mtbench", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "truncated": 21 |
| }, |
| "sb_coding": { |
| "acc_len": 4.262703790885098, |
| "acc_len_max": 5.730769230769231, |
| "acc_len_median": 4.396610169491526, |
| "acc_len_min": 2.6, |
| "concurrency": 1, |
| "elapsed_s": 774.4, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 1405, |
| "mean_prompt_tokens": 405, |
| "n_acc_matched": 89, |
| "n_acc_unmatched": 0, |
| "n_prompts": 80, |
| "n_prompts_expected": 80, |
| "n_requests": 89, |
| "output_tokens": 125028, |
| "output_tokens_per_s": 161.452, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.466100541555014, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "sb_coding", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "truncated": 3 |
| }, |
| "sb_multilingual": { |
| "acc_len": 3.559084746910859, |
| "acc_len_max": 5.517857142857143, |
| "acc_len_median": 3.340179316096747, |
| "acc_len_min": 2.4008875739644973, |
| "concurrency": 1, |
| "elapsed_s": 297.4, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 586, |
| "mean_prompt_tokens": 179, |
| "n_acc_matched": 80, |
| "n_acc_unmatched": 0, |
| "n_prompts": 80, |
| "n_prompts_expected": 80, |
| "n_requests": 80, |
| "output_tokens": 46892, |
| "output_tokens_per_s": 157.65, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.3655835352729798, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "sb_multilingual", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "truncated": 0 |
| }, |
| "sb_qa": { |
| "acc_len": 3.1203881130019955, |
| "acc_len_max": 4.268456375838926, |
| "acc_len_median": 3.0877691387559807, |
| "acc_len_min": 2.2061971830985914, |
| "concurrency": 1, |
| "elapsed_s": 597.6, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 985, |
| "mean_prompt_tokens": 99, |
| "n_acc_matched": 80, |
| "n_acc_unmatched": 0, |
| "n_prompts": 80, |
| "n_prompts_expected": 80, |
| "n_requests": 80, |
| "output_tokens": 78817, |
| "output_tokens_per_s": 131.898, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.30291258757171363, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "sb_qa", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "truncated": 2 |
| }, |
| "sb_rag": { |
| "acc_len": 3.8188619710184173, |
| "acc_len_max": 5.541666666666667, |
| "acc_len_median": 3.7484909456740443, |
| "acc_len_min": 2.709433962264151, |
| "concurrency": 1, |
| "elapsed_s": 427.5, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 650, |
| "mean_prompt_tokens": 703, |
| "n_acc_matched": 91, |
| "n_acc_unmatched": 0, |
| "n_prompts": 80, |
| "n_prompts_expected": 80, |
| "n_requests": 91, |
| "output_tokens": 59139, |
| "output_tokens_per_s": 138.34, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.40269456728834535, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "sb_rag", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "truncated": 1 |
| }, |
| "sb_writing": { |
| "acc_len": 2.676241748033513, |
| "acc_len_max": 4.743975903614458, |
| "acc_len_median": 2.5655032035301577, |
| "acc_len_min": 1.941678520625889, |
| "concurrency": 1, |
| "elapsed_s": 3406.6, |
| "errors": 0, |
| "max_tokens": 4096, |
| "mean_completion_tokens": 3543, |
| "mean_prompt_tokens": 785, |
| "n_acc_matched": 84, |
| "n_acc_unmatched": 0, |
| "n_prompts": 80, |
| "n_prompts_expected": 80, |
| "n_requests": 84, |
| "output_tokens": 297601, |
| "output_tokens_per_s": 87.359, |
| "prompt_policy": "raw dataset text, no instruction wrapper", |
| "proposal_acceptance": 0.23946310686193042, |
| "proposal_count": 7, |
| "reasoning": "max", |
| "row": "sb_writing", |
| "sampling": { |
| "temperature": 1.0, |
| "top_p": 0.95 |
| }, |
| "truncated": 54 |
| } |
| } |
