codedebugger / outputs /baselines_20260425_102053.json
psdhanushkumar's picture
Upload folder using huggingface_hub
f361447 verified
Raw
History Blame Contribute Delete
23 kB
{
"timestamp": "2026-04-25T10:19:18.749709",
"baselines": {
"no_change": {
"baseline": "no_change",
"total_problems": 30,
"solved": 4,
"solve_rate": 0.1333,
"avg_reward": 0.5858,
"avg_steps": 1.0,
"avg_pass_rate": 0.3833,
"results": [
{
"problem_id": "bug_001",
"title": "Off-by-one in list index",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.3,
"solved": false
},
{
"problem_id": "bug_002",
"title": "Wrong comparison operator (> instead of >=)",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.7125,
"solved": false
},
{
"problem_id": "bug_003",
"title": "Missing return statement",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_004",
"title": "Wrong variable name used",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_005",
"title": "Wrong string method: split vs strip",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_006",
"title": "Missing -1 in length calculation",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_007",
"title": "Wrong list method: append vs extend",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_008",
"title": "Comparing to None with == instead of is",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_009",
"title": "Integer division instead of float division",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_010",
"title": "Wrong list index: -1 vs 0",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_011",
"title": "Wrong loop termination condition",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_012",
"title": "Wrong recursion base case value",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_013",
"title": "Wrong sort order (missing reverse=True)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_014",
"title": "Missing edge case for empty list",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.5625,
"solved": false
},
{
"problem_id": "bug_015",
"title": "Wrong string format (% vs .format)",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.3,
"solved": false
},
{
"problem_id": "bug_016",
"title": "Wrong dictionary key access",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.475,
"solved": false
},
{
"problem_id": "bug_017",
"title": "Wrong boolean logic (and vs or)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_018",
"title": "Wrong range bounds (range(n) vs range(n+1))",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_019",
"title": "Missing int() type conversion",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.45,
"solved": false
},
{
"problem_id": "bug_020",
"title": "Wrong list slicing ([1:] vs [:1])",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_021",
"title": "Binary search off-by-one in mid update",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.5625,
"solved": false
},
{
"problem_id": "bug_022",
"title": "Wrong DP state transition (Fibonacci)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_023",
"title": "Wrong graph traversal: visited check in wrong place",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_024",
"title": "Missing memoization causing exponential recursion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 1.0,
"solved": true
},
{
"problem_id": "bug_025",
"title": "Wrong two-pointer movement direction",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_026",
"title": "Wrong greedy choice (min vs max)",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_027",
"title": "Off-by-one in sliding window boundary",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_028",
"title": "Wrong merge logic in merge sort",
"steps_taken": 1,
"best_pass_rate": 0.5,
"cumulative_reward": 0.625,
"solved": false
},
{
"problem_id": "bug_029",
"title": "Wrong heap operation (push vs pop)",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
},
{
"problem_id": "bug_030",
"title": "Wrong backtracking: missing undo step",
"steps_taken": 1,
"best_pass_rate": 0.25,
"cumulative_reward": 0.5375,
"solved": false
}
]
},
"oracle": {
"baseline": "oracle",
"total_problems": 30,
"solved": 26,
"solve_rate": 0.8667,
"avg_reward": 0.9363,
"avg_steps": 1.0,
"avg_pass_rate": 0.9167,
"results": [
{
"problem_id": "bug_001",
"title": "Off-by-one in list index",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9957,
"solved": true
},
{
"problem_id": "bug_002",
"title": "Wrong comparison operator (> instead of >=)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9989,
"solved": true
},
{
"problem_id": "bug_003",
"title": "Missing return statement",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9791,
"solved": true
},
{
"problem_id": "bug_004",
"title": "Wrong variable name used",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9985,
"solved": true
},
{
"problem_id": "bug_005",
"title": "Wrong string method: split vs strip",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.4414,
"solved": false
},
{
"problem_id": "bug_006",
"title": "Missing -1 in length calculation",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9981,
"solved": true
},
{
"problem_id": "bug_007",
"title": "Wrong list method: append vs extend",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9954,
"solved": true
},
{
"problem_id": "bug_008",
"title": "Comparing to None with == instead of is",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9952,
"solved": true
},
{
"problem_id": "bug_009",
"title": "Integer division instead of float division",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.999,
"solved": true
},
{
"problem_id": "bug_010",
"title": "Wrong list index: -1 vs 0",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9961,
"solved": true
},
{
"problem_id": "bug_011",
"title": "Wrong loop termination condition",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9986,
"solved": true
},
{
"problem_id": "bug_012",
"title": "Wrong recursion base case value",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9988,
"solved": true
},
{
"problem_id": "bug_013",
"title": "Wrong sort order (missing reverse=True)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.986,
"solved": true
},
{
"problem_id": "bug_014",
"title": "Missing edge case for empty list",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9684,
"solved": true
},
{
"problem_id": "bug_015",
"title": "Wrong string format (% vs .format)",
"steps_taken": 1,
"best_pass_rate": 0.0,
"cumulative_reward": 0.4363,
"solved": false
},
{
"problem_id": "bug_016",
"title": "Wrong dictionary key access",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9848,
"solved": true
},
{
"problem_id": "bug_017",
"title": "Wrong boolean logic (and vs or)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9937,
"solved": true
},
{
"problem_id": "bug_018",
"title": "Wrong range bounds (range(n) vs range(n+1))",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9955,
"solved": true
},
{
"problem_id": "bug_019",
"title": "Missing int() type conversion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9935,
"solved": true
},
{
"problem_id": "bug_020",
"title": "Wrong list slicing ([1:] vs [:1])",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9971,
"solved": true
},
{
"problem_id": "bug_021",
"title": "Binary search off-by-one in mid update",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9985,
"solved": true
},
{
"problem_id": "bug_022",
"title": "Wrong DP state transition (Fibonacci)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9994,
"solved": true
},
{
"problem_id": "bug_023",
"title": "Wrong graph traversal: visited check in wrong place",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.969,
"solved": true
},
{
"problem_id": "bug_024",
"title": "Missing memoization causing exponential recursion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9733,
"solved": true
},
{
"problem_id": "bug_025",
"title": "Wrong two-pointer movement direction",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.6992,
"solved": false
},
{
"problem_id": "bug_026",
"title": "Wrong greedy choice (min vs max)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9963,
"solved": true
},
{
"problem_id": "bug_027",
"title": "Off-by-one in sliding window boundary",
"steps_taken": 1,
"best_pass_rate": 0.75,
"cumulative_reward": 0.7115,
"solved": false
},
{
"problem_id": "bug_028",
"title": "Wrong merge logic in merge sort",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9995,
"solved": true
},
{
"problem_id": "bug_029",
"title": "Wrong heap operation (push vs pop)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9943,
"solved": true
},
{
"problem_id": "bug_030",
"title": "Wrong backtracking: missing undo step",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9968,
"solved": true
}
]
},
"llm": {
"baseline": "llm_llama-3.3-70b-versatile",
"total_problems": 30,
"solved": 27,
"solve_rate": 0.9,
"avg_reward": 1.1046,
"avg_steps": 1.37,
"avg_pass_rate": 0.9667,
"results": [
{
"problem_id": "bug_001",
"title": "Off-by-one in list index",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9647,
"solved": true
},
{
"problem_id": "bug_002",
"title": "Wrong comparison operator (> instead of >=)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9338,
"solved": true
},
{
"problem_id": "bug_003",
"title": "Missing return statement",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.98,
"solved": true
},
{
"problem_id": "bug_004",
"title": "Wrong variable name used",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9977,
"solved": true
},
{
"problem_id": "bug_005",
"title": "Wrong string method: split vs strip",
"steps_taken": 3,
"best_pass_rate": 1.0,
"cumulative_reward": 1.56,
"solved": true
},
{
"problem_id": "bug_006",
"title": "Missing -1 in length calculation",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9441,
"solved": true
},
{
"problem_id": "bug_007",
"title": "Wrong list method: append vs extend",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9533,
"solved": true
},
{
"problem_id": "bug_008",
"title": "Comparing to None with == instead of is",
"steps_taken": 3,
"best_pass_rate": 0.5,
"cumulative_reward": 1.498,
"solved": false
},
{
"problem_id": "bug_009",
"title": "Integer division instead of float division",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.942,
"solved": true
},
{
"problem_id": "bug_010",
"title": "Wrong list index: -1 vs 0",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9605,
"solved": true
},
{
"problem_id": "bug_011",
"title": "Wrong loop termination condition",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.94,
"solved": true
},
{
"problem_id": "bug_012",
"title": "Wrong recursion base case value",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9459,
"solved": true
},
{
"problem_id": "bug_013",
"title": "Wrong sort order (missing reverse=True)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9469,
"solved": true
},
{
"problem_id": "bug_014",
"title": "Missing edge case for empty list",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9388,
"solved": true
},
{
"problem_id": "bug_015",
"title": "Wrong string format (% vs .format)",
"steps_taken": 2,
"best_pass_rate": 1.0,
"cumulative_reward": 1.409,
"solved": true
},
{
"problem_id": "bug_016",
"title": "Wrong dictionary key access",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9539,
"solved": true
},
{
"problem_id": "bug_017",
"title": "Wrong boolean logic (and vs or)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.949,
"solved": true
},
{
"problem_id": "bug_018",
"title": "Wrong range bounds (range(n) vs range(n+1))",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9312,
"solved": true
},
{
"problem_id": "bug_019",
"title": "Missing int() type conversion",
"steps_taken": 2,
"best_pass_rate": 1.0,
"cumulative_reward": 1.3108,
"solved": true
},
{
"problem_id": "bug_020",
"title": "Wrong list slicing ([1:] vs [:1])",
"steps_taken": 2,
"best_pass_rate": 1.0,
"cumulative_reward": 1.4,
"solved": true
},
{
"problem_id": "bug_021",
"title": "Binary search off-by-one in mid update",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9853,
"solved": true
},
{
"problem_id": "bug_022",
"title": "Wrong DP state transition (Fibonacci)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9898,
"solved": true
},
{
"problem_id": "bug_023",
"title": "Wrong graph traversal: visited check in wrong place",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9804,
"solved": true
},
{
"problem_id": "bug_024",
"title": "Missing memoization causing exponential recursion",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9418,
"solved": true
},
{
"problem_id": "bug_025",
"title": "Wrong two-pointer movement direction",
"steps_taken": 3,
"best_pass_rate": 0.75,
"cumulative_reward": 1.9622,
"solved": false
},
{
"problem_id": "bug_026",
"title": "Wrong greedy choice (min vs max)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9812,
"solved": true
},
{
"problem_id": "bug_027",
"title": "Off-by-one in sliding window boundary",
"steps_taken": 3,
"best_pass_rate": 0.75,
"cumulative_reward": 1.877,
"solved": false
},
{
"problem_id": "bug_028",
"title": "Wrong merge logic in merge sort",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.9856,
"solved": true
},
{
"problem_id": "bug_029",
"title": "Wrong heap operation (push vs pop)",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.994,
"solved": true
},
{
"problem_id": "bug_030",
"title": "Wrong backtracking: missing undo step",
"steps_taken": 1,
"best_pass_rate": 1.0,
"cumulative_reward": 0.982,
"solved": true
}
]
}
}
}