codedebugger / outputs /baselines_20260425_102451.json
psdhanushkumar's picture
Upload folder using huggingface_hub
f361447 verified
Raw
History Blame Contribute Delete
23 kB
{
"timestamp": "2026-04-25T10:23:42.599050",
"baselines": {
"no_change": {
"baseline": "no_change",
"total_problems": 30,
"solved": 4,
"solve_rate": 0.1333,
"avg_reward": 0.5887,
"avg_steps": 1.0,
"avg_pass_rate": 0.3917,
"results": [
{
"problem_id": "bug_001",
"title": "Off-by-one in list index",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.3,
"solved": false
},
{
"problem_id": "bug_002",
"title": "Wrong comparison operator (> instead of >=)",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.7125,
"solved": false
},
{
"problem_id": "bug_003",
"title": "Missing return statement",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_004",
"title": "Wrong variable name used",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_005",
"title": "Wrong string method: split vs strip",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_006",
"title": "Missing -1 in length calculation",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_007",
"title": "Wrong list method: append vs extend",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_008",
"title": "Comparing to None with == instead of is",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_009",
"title": "Integer division instead of float division",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_010",
"title": "Wrong list index: -1 vs 0",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_011",
"title": "Wrong loop termination condition",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_012",
"title": "Wrong recursion base case value",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_013",
"title": "Wrong sort order (missing reverse=True)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_014",
"title": "Missing edge case for empty list",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.5625,
"solved": false
},
{
"problem_id": "bug_015",
"title": "Wrong string format (% vs .format)",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.3,
"solved": false
},
{
"problem_id": "bug_016",
"title": "Wrong dictionary key access",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.475,
"solved": false
},
{
"problem_id": "bug_017",
"title": "Wrong boolean logic (and vs or)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_018",
"title": "Wrong range bounds (range(n) vs range(n+1))",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_019",
"title": "Missing int() type conversion",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_020",
"title": "Wrong list slicing ([1:] vs [:1])",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_021",
"title": "Binary search off-by-one in mid update",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.5625,
"solved": false
},
{
"problem_id": "bug_022",
"title": "Wrong DP state transition (Fibonacci)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_023",
"title": "Wrong graph traversal: visited check in wrong place",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_024",
"title": "Missing memoization causing exponential recursion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_025",
"title": "Wrong two-pointer movement direction",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.7125,
"solved": false
},
{
"problem_id": "bug_026",
"title": "Wrong greedy choice (min vs max)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_027",
"title": "Off-by-one in sliding window boundary",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_028",
"title": "Wrong merge logic in merge sort",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_029",
"title": "Wrong heap operation (push vs pop)",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_030",
"title": "Wrong backtracking: missing undo step",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
}
]
},
"oracle": {
"baseline": "oracle",
"total_problems": 30,
"solved": 30,
"solve_rate": 1.0,
"avg_reward": 0.9921,
"avg_steps": 1.0,
"avg_pass_rate": 1.0,
"results": [
{
"problem_id": "bug_001",
"title": "Off-by-one in list index",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9957,
"solved": true
},
{
"problem_id": "bug_002",
"title": "Wrong comparison operator (> instead of >=)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9989,
"solved": true
},
{
"problem_id": "bug_003",
"title": "Missing return statement",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9791,
"solved": true
},
{
"problem_id": "bug_004",
"title": "Wrong variable name used",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9985,
"solved": true
},
{
"problem_id": "bug_005",
"title": "Wrong string method: split vs strip",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9914,
"solved": true
},
{
"problem_id": "bug_006",
"title": "Missing -1 in length calculation",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9981,
"solved": true
},
{
"problem_id": "bug_007",
"title": "Wrong list method: append vs extend",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9954,
"solved": true
},
{
"problem_id": "bug_008",
"title": "Comparing to None with == instead of is",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9952,
"solved": true
},
{
"problem_id": "bug_009",
"title": "Integer division instead of float division",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.999,
"solved": true
},
{
"problem_id": "bug_010",
"title": "Wrong list index: -1 vs 0",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9961,
"solved": true
},
{
"problem_id": "bug_011",
"title": "Wrong loop termination condition",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9986,
"solved": true
},
{
"problem_id": "bug_012",
"title": "Wrong recursion base case value",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9988,
"solved": true
},
{
"problem_id": "bug_013",
"title": "Wrong sort order (missing reverse=True)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.986,
"solved": true
},
{
"problem_id": "bug_014",
"title": "Missing edge case for empty list",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9684,
"solved": true
},
{
"problem_id": "bug_015",
"title": "Wrong string format (% vs .format)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9863,
"solved": true
},
{
"problem_id": "bug_016",
"title": "Wrong dictionary key access",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9848,
"solved": true
},
{
"problem_id": "bug_017",
"title": "Wrong boolean logic (and vs or)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9937,
"solved": true
},
{
"problem_id": "bug_018",
"title": "Wrong range bounds (range(n) vs range(n+1))",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9955,
"solved": true
},
{
"problem_id": "bug_019",
"title": "Missing int() type conversion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9935,
"solved": true
},
{
"problem_id": "bug_020",
"title": "Wrong list slicing ([1:] vs [:1])",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9971,
"solved": true
},
{
"problem_id": "bug_021",
"title": "Binary search off-by-one in mid update",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9985,
"solved": true
},
{
"problem_id": "bug_022",
"title": "Wrong DP state transition (Fibonacci)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9994,
"solved": true
},
{
"problem_id": "bug_023",
"title": "Wrong graph traversal: visited check in wrong place",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.969,
"solved": true
},
{
"problem_id": "bug_024",
"title": "Missing memoization causing exponential recursion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9733,
"solved": true
},
{
"problem_id": "bug_025",
"title": "Wrong two-pointer movement direction",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9867,
"solved": true
},
{
"problem_id": "bug_026",
"title": "Wrong greedy choice (min vs max)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9963,
"solved": true
},
{
"problem_id": "bug_027",
"title": "Off-by-one in sliding window boundary",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.999,
"solved": true
},
{
"problem_id": "bug_028",
"title": "Wrong merge logic in merge sort",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9995,
"solved": true
},
{
"problem_id": "bug_029",
"title": "Wrong heap operation (push vs pop)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9943,
"solved": true
},
{
"problem_id": "bug_030",
"title": "Wrong backtracking: missing undo step",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9968,
"solved": true
}
]
},
"llm": {
"baseline": "llm_llama-3.3-70b-versatile",
"total_problems": 30,
"solved": 29,
"solve_rate": 0.9667,
"avg_reward": 1.0056,
"avg_steps": 1.13,
"avg_pass_rate": 0.9833,
"results": [
{
"problem_id": "bug_001",
"title": "Off-by-one in list index",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9391,
"solved": true
},
{
"problem_id": "bug_002",
"title": "Wrong comparison operator (> instead of >=)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9682,
"solved": true
},
{
"problem_id": "bug_003",
"title": "Missing return statement",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.98,
"solved": true
},
{
"problem_id": "bug_004",
"title": "Wrong variable name used",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9977,
"solved": true
},
{
"problem_id": "bug_005",
"title": "Wrong string method: split vs strip",
"steps_taken": 2,
"best_pass_rate": 1.0,
"cumulative_reward": 1.2898,
"solved": true
},
{
"problem_id": "bug_006",
"title": "Missing -1 in length calculation",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9448,
"solved": true
},
{
"problem_id": "bug_007",
"title": "Wrong list method: append vs extend",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9537,
"solved": true
},
{
"problem_id": "bug_008",
"title": "Comparing to None with == instead of is",
"steps_taken": 3,
"best_pass_rate": 0.5,
"cumulative_reward": 1.5463,
"solved": false
},
{
"problem_id": "bug_009",
"title": "Integer division instead of float division",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9295,
"solved": true
},
{
"problem_id": "bug_010",
"title": "Wrong list index: -1 vs 0",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9605,
"solved": true
},
{
"problem_id": "bug_011",
"title": "Wrong loop termination condition",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.941,
"solved": true
},
{
"problem_id": "bug_012",
"title": "Wrong recursion base case value",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9755,
"solved": true
},
{
"problem_id": "bug_013",
"title": "Wrong sort order (missing reverse=True)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9519,
"solved": true
},
{
"problem_id": "bug_014",
"title": "Missing edge case for empty list",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9388,
"solved": true
},
{
"problem_id": "bug_015",
"title": "Wrong string format (% vs .format)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9968,
"solved": true
},
{
"problem_id": "bug_016",
"title": "Wrong dictionary key access",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9231,
"solved": true
},
{
"problem_id": "bug_017",
"title": "Wrong boolean logic (and vs or)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.949,
"solved": true
},
{
"problem_id": "bug_018",
"title": "Wrong range bounds (range(n) vs range(n+1))",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9402,
"solved": true
},
{
"problem_id": "bug_019",
"title": "Missing int() type conversion",
"steps_taken": 2,
"best_pass_rate": 1.0,
"cumulative_reward": 1.3067,
"solved": true
},
{
"problem_id": "bug_020",
"title": "Wrong list slicing ([1:] vs [:1])",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9561,
"solved": true
},
{
"problem_id": "bug_021",
"title": "Binary search off-by-one in mid update",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9931,
"solved": true
},
{
"problem_id": "bug_022",
"title": "Wrong DP state transition (Fibonacci)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9888,
"solved": true
},
{
"problem_id": "bug_023",
"title": "Wrong graph traversal: visited check in wrong place",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9794,
"solved": true
},
{
"problem_id": "bug_024",
"title": "Missing memoization causing exponential recursion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9736,
"solved": true
},
{
"problem_id": "bug_025",
"title": "Wrong two-pointer movement direction",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.974,
"solved": true
},
{
"problem_id": "bug_026",
"title": "Wrong greedy choice (min vs max)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9812,
"solved": true
},
{
"problem_id": "bug_027",
"title": "Off-by-one in sliding window boundary",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9908,
"solved": true
},
{
"problem_id": "bug_028",
"title": "Wrong merge logic in merge sort",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9877,
"solved": true
},
{
"problem_id": "bug_029",
"title": "Wrong heap operation (push vs pop)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.932,
"solved": true
},
{
"problem_id": "bug_030",
"title": "Wrong backtracking: missing undo step",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9775,
"solved": true
}
]
}
}
}