| {"models": [{"name": "GLM-5.2:together", "total": {"n": 592, "acc": 58.6, "stop_reasons": {"solution_found": 345, "timeout": 247}, "med_output": 600, "med_output_ci": [475, 804], "med_turns": 3.0, "med_turns_ci": [3.0, 3.0]}, "by_language": {"java": {"n": 143, "acc": 58.0, "stop_reasons": {"solution_found": 84, "timeout": 59}, "med_output": 619, "med_output_ci": [392, 899], "med_turns": 3, "med_turns_ci": [2, 3]}, "ocaml": {"n": 161, "acc": 59.0, "stop_reasons": {"solution_found": 94, "timeout": 67}, "med_output": 669, "med_output_ci": [479, 1368], "med_turns": 4, "med_turns_ci": [3, 4]}, "python": {"n": 151, "acc": 56.3, "stop_reasons": {"solution_found": 83, "timeout": 68}, "med_output": 593, "med_output_ci": [343, 866], "med_turns": 3, "med_turns_ci": [3, 3]}, "rust": {"n": 137, "acc": 61.3, "stop_reasons": {"solution_found": 84, "timeout": 53}, "med_output": 543, "med_output_ci": [389, 891], "med_turns": 3, "med_turns_ci": [3, 4]}}}, {"name": "claude-sonnet-4-6", "total": {"n": 400, "acc": 86.2, "stop_reasons": {"solution_found": 347, "timeout": 50, "turn_limit": 3}, "med_output": 1990, "med_output_ci": [1538, 2544], "med_turns": 3.0, "med_turns_ci": [3.0, 4.0]}, "by_language": {"java": {"n": 100, "acc": 86.0, "stop_reasons": {"solution_found": 87, "timeout": 12, "turn_limit": 1}, "med_output": 2008, "med_output_ci": [1018, 3488], "med_turns": 3.0, "med_turns_ci": [2.0, 4.0]}, "ocaml": {"n": 100, "acc": 86.0, "stop_reasons": {"solution_found": 87, "timeout": 12, "turn_limit": 1}, "med_output": 2218, "med_output_ci": [1260, 3982], "med_turns": 4.0, "med_turns_ci": [3.0, 5.0]}, "python": {"n": 100, "acc": 87.0, "stop_reasons": {"solution_found": 86, "timeout": 13, "turn_limit": 1}, "med_output": 1604, "med_output_ci": [812, 3324], "med_turns": 3.0, "med_turns_ci": [3.0, 4.0]}, "rust": {"n": 100, "acc": 86.0, "stop_reasons": {"solution_found": 87, "timeout": 13}, "med_output": 1892, "med_output_ci": [1116, 3123], "med_turns": 4.0, "med_turns_ci": [3.0, 5.0]}}}, {"name": "gemma_4_26b_a4b_it", "total": {"n": 400, "acc": 75.5, "stop_reasons": {"solution_found": 288, "timeout": 87, "turn_limit": 25}, "med_output": 16961, "med_output_ci": [12471, 21342], "med_turns": 5.0, "med_turns_ci": [5.0, 6.0]}, "by_language": {"java": {"n": 100, "acc": 81.0, "stop_reasons": {"solution_found": 81, "timeout": 18, "turn_limit": 1}, "med_output": 12007, "med_output_ci": [7697, 19395], "med_turns": 3.5, "med_turns_ci": [3.0, 4.0]}, "ocaml": {"n": 100, "acc": 54.0, "stop_reasons": {"solution_found": 48, "timeout": 29, "turn_limit": 23}, "med_output": 23097, "med_output_ci": [16376, 29166], "med_turns": 14.5, "med_turns_ci": [11.0, 20.5]}, "python": {"n": 100, "acc": 81.0, "stop_reasons": {"solution_found": 82, "timeout": 18}, "med_output": 12704, "med_output_ci": [6246, 30319], "med_turns": 5.0, "med_turns_ci": [4.0, 7.5]}, "rust": {"n": 100, "acc": 86.0, "stop_reasons": {"solution_found": 77, "timeout": 22, "turn_limit": 1}, "med_output": 15394, "med_output_ci": [9208, 25099], "med_turns": 5.0, "med_turns_ci": [4.5, 7.0]}}}, {"name": "gpt-5.5", "total": {"n": 400, "acc": 91.8, "stop_reasons": {"solution_found": 371, "timeout": 29}, "med_output": 923, "med_output_ci": [787, 1087], "med_turns": 3.0, "med_turns_ci": [3.0, 3.0]}, "by_language": {"java": {"n": 100, "acc": 93.0, "stop_reasons": {"solution_found": 94, "timeout": 6}, "med_output": 1009, "med_output_ci": [783, 1278], "med_turns": 3.0, "med_turns_ci": [3.0, 3.0]}, "ocaml": {"n": 100, "acc": 90.0, "stop_reasons": {"solution_found": 91, "timeout": 9}, "med_output": 1169, "med_output_ci": [862, 1531], "med_turns": 3.0, "med_turns_ci": [3.0, 4.0]}, "python": {"n": 100, "acc": 92.0, "stop_reasons": {"solution_found": 93, "timeout": 7}, "med_output": 696, "med_output_ci": [525, 1014], "med_turns": 3.0, "med_turns_ci": [3.0, 3.0]}, "rust": {"n": 100, "acc": 92.0, "stop_reasons": {"solution_found": 93, "timeout": 7}, "med_output": 902, "med_output_ci": [757, 1173], "med_turns": 4.0, "med_turns_ci": [4.0, 4.0]}}}, {"name": "qwen3p6_35b_a3b_fp8", "total": {"n": 400, "acc": 77.2, "stop_reasons": {"solution_found": 308, "timeout": 78, "turn_limit": 14}, "med_output": 4548, "med_output_ci": [3444, 6952], "med_turns": 8.0, "med_turns_ci": [7.0, 9.0]}, "by_language": {"java": {"n": 100, "acc": 76.0, "stop_reasons": {"solution_found": 76, "timeout": 23, "turn_limit": 1}, "med_output": 3444, "med_output_ci": [2550, 7112], "med_turns": 7.0, "med_turns_ci": [6.0, 8.0]}, "ocaml": {"n": 100, "acc": 72.0, "stop_reasons": {"solution_found": 70, "timeout": 18, "turn_limit": 12}, "med_output": 9557, "med_output_ci": [5074, 15190], "med_turns": 13.0, "med_turns_ci": [11.0, 18.0]}, "python": {"n": 100, "acc": 80.0, "stop_reasons": {"solution_found": 82, "timeout": 18}, "med_output": 3142, "med_output_ci": [2080, 6875], "med_turns": 6.0, "med_turns_ci": [5.0, 7.0]}, "rust": {"n": 100, "acc": 81.0, "stop_reasons": {"solution_found": 80, "timeout": 19, "turn_limit": 1}, "med_output": 4503, "med_output_ci": [3036, 7395], "med_turns": 8.0, "med_turns_ci": [7.0, 9.0]}}}], "total_files": 2392} |