Spaces:
Sleeping
Sleeping
| { | |
| "timestamp": "2026-04-25T10:07:45.034726", | |
| "baselines": { | |
| "no_change": { | |
| "baseline": "no_change", | |
| "total_problems": 30, | |
| "solved": 1, | |
| "solve_rate": 0.0333, | |
| "avg_reward": 0.5503, | |
| "avg_steps": 1.0, | |
| "avg_pass_rate": 0.3056, | |
| "results": [ | |
| { | |
| "problem_id": "bug_001", | |
| "title": "Off-by-one in range", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_002", | |
| "title": "Wrong default mutable argument", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_003", | |
| "title": "Integer division instead of float", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_004", | |
| "title": "Missing return statement", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_005", | |
| "title": "String concatenation type error", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_006", | |
| "title": "Wrong comparison operator", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.2, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_007", | |
| "title": "Index out of range in list access", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_008", | |
| "title": "Infinite recursion \u2014 missing base case", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_009", | |
| "title": "Wrong variable in return", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_010", | |
| "title": "Dictionary KeyError", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.3333333333333333, | |
| "cumulative_reward": 0.5667, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_011", | |
| "title": "Reversed sort order", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_012", | |
| "title": "Mutating input list unintentionally", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_013", | |
| "title": "Forgotten self in method", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.3333333333333333, | |
| "cumulative_reward": 0.5667, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_014", | |
| "title": "Wrong boolean logic", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.3333333333333333, | |
| "cumulative_reward": 0.5667, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_015", | |
| "title": "Scope issue with global variable", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_016", | |
| "title": "Shallow copy issue", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.5, | |
| "cumulative_reward": 0.625, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_017", | |
| "title": "Wrong string method", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_018", | |
| "title": "Wrong exception handling", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_019", | |
| "title": "Fibonacci off-by-one", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 1.0, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_020", | |
| "title": "List comprehension filter wrong", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_021", | |
| "title": "Binary search wrong midpoint update", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.3, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_022", | |
| "title": "Reversed dictionary comprehension", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.3333333333333333, | |
| "cumulative_reward": 0.5667, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_023", | |
| "title": "String reverse off by one", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.3333333333333333, | |
| "cumulative_reward": 0.5667, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_024", | |
| "title": "Wrong accumulator init for max", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_025", | |
| "title": "Nested loop wrong variable", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_026", | |
| "title": "Palindrome check wrong slice", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_027", | |
| "title": "Power function wrong operator", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_028", | |
| "title": "GCD wrong recursion", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.0, | |
| "cumulative_reward": 0.45, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_029", | |
| "title": "Zip truncation issue", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_030", | |
| "title": "Matrix transpose wrong indices", | |
| "steps_taken": 1, | |
| "best_pass_rate": 0.6666666666666666, | |
| "cumulative_reward": 0.6833, | |
| "solved": false | |
| } | |
| ] | |
| }, | |
| "oracle": { | |
| "baseline": "oracle", | |
| "total_problems": 30, | |
| "solved": 30, | |
| "solve_rate": 1.0, | |
| "avg_reward": 0.9904, | |
| "avg_steps": 1.0, | |
| "avg_pass_rate": 1.0, | |
| "results": [ | |
| { | |
| "problem_id": "bug_001", | |
| "title": "Off-by-one in range", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9978, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_002", | |
| "title": "Wrong default mutable argument", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9751, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_003", | |
| "title": "Integer division instead of float", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.999, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_004", | |
| "title": "Missing return statement", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9786, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_005", | |
| "title": "String concatenation type error", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9971, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_006", | |
| "title": "Wrong comparison operator", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9986, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_007", | |
| "title": "Index out of range in list access", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9959, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_008", | |
| "title": "Infinite recursion \u2014 missing base case", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9754, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_009", | |
| "title": "Wrong variable in return", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9985, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_010", | |
| "title": "Dictionary KeyError", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9877, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_011", | |
| "title": "Reversed sort order", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9873, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_012", | |
| "title": "Mutating input list unintentionally", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9611, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_013", | |
| "title": "Forgotten self in method", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9984, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_014", | |
| "title": "Wrong boolean logic", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9954, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_015", | |
| "title": "Scope issue with global variable", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9831, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_016", | |
| "title": "Shallow copy issue", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9977, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_017", | |
| "title": "Wrong string method", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9966, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_018", | |
| "title": "Wrong exception handling", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9926, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_019", | |
| "title": "Fibonacci off-by-one", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9981, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_020", | |
| "title": "List comprehension filter wrong", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9984, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_021", | |
| "title": "Binary search wrong midpoint update", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9985, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_022", | |
| "title": "Reversed dictionary comprehension", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9966, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_023", | |
| "title": "String reverse off by one", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9921, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_024", | |
| "title": "Wrong accumulator init for max", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9964, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_025", | |
| "title": "Nested loop wrong variable", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9974, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_026", | |
| "title": "Palindrome check wrong slice", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.983, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_027", | |
| "title": "Power function wrong operator", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9966, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_028", | |
| "title": "GCD wrong recursion", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9986, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_029", | |
| "title": "Zip truncation issue", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9551, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_030", | |
| "title": "Matrix transpose wrong indices", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9858, | |
| "solved": true | |
| } | |
| ] | |
| }, | |
| "llm": { | |
| "baseline": "llm_llama-3.3-70b-versatile", | |
| "total_problems": 30, | |
| "solved": 29, | |
| "solve_rate": 0.9667, | |
| "avg_reward": 1.0346, | |
| "avg_steps": 1.17, | |
| "avg_pass_rate": 0.9833, | |
| "results": [ | |
| { | |
| "problem_id": "bug_001", | |
| "title": "Off-by-one in range", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9963, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_002", | |
| "title": "Wrong default mutable argument", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9751, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_003", | |
| "title": "Integer division instead of float", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9548, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_004", | |
| "title": "Missing return statement", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9786, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_005", | |
| "title": "String concatenation type error", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9971, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_006", | |
| "title": "Wrong comparison operator", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9986, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_007", | |
| "title": "Index out of range in list access", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9448, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_008", | |
| "title": "Infinite recursion \u2014 missing base case", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9473, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_009", | |
| "title": "Wrong variable in return", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9985, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_010", | |
| "title": "Dictionary KeyError", | |
| "steps_taken": 2, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 1.4187, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_011", | |
| "title": "Reversed sort order", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9568, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_012", | |
| "title": "Mutating input list unintentionally", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.968, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_013", | |
| "title": "Forgotten self in method", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9886, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_014", | |
| "title": "Wrong boolean logic", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9954, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_015", | |
| "title": "Scope issue with global variable", | |
| "steps_taken": 3, | |
| "best_pass_rate": 0.5, | |
| "cumulative_reward": 1.1811, | |
| "solved": false | |
| }, | |
| { | |
| "problem_id": "bug_016", | |
| "title": "Shallow copy issue", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9656, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_017", | |
| "title": "Wrong string method", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9966, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_018", | |
| "title": "Wrong exception handling", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9897, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_019", | |
| "title": "Fibonacci off-by-one", | |
| "steps_taken": 3, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 2.0001, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_020", | |
| "title": "List comprehension filter wrong", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9984, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_021", | |
| "title": "Binary search wrong midpoint update", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9901, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_022", | |
| "title": "Reversed dictionary comprehension", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9651, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_023", | |
| "title": "String reverse off by one", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9921, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_024", | |
| "title": "Wrong accumulator init for max", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9719, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_025", | |
| "title": "Nested loop wrong variable", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9883, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_026", | |
| "title": "Palindrome check wrong slice", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.983, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_027", | |
| "title": "Power function wrong operator", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9966, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_028", | |
| "title": "GCD wrong recursion", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9567, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_029", | |
| "title": "Zip truncation issue", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9768, | |
| "solved": true | |
| }, | |
| { | |
| "problem_id": "bug_030", | |
| "title": "Matrix transpose wrong indices", | |
| "steps_taken": 1, | |
| "best_pass_rate": 1.0, | |
| "cumulative_reward": 0.9677, | |
| "solved": true | |
| } | |
| ] | |
| } | |
| } | |
| } |