{
  "timestamp": "2026-04-25T10:19:18.749709",
  "baselines": {
    "no_change": {
      "baseline": "no_change",
      "total_problems": 30,
      "solved": 4,
      "solve_rate": 0.1333,
      "avg_reward": 0.5858,
      "avg_steps": 1.0,
      "avg_pass_rate": 0.3833,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in list index",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.3,
          "solved": false
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong comparison operator (> instead of >=)",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.7125,
          "solved": false
        },
        {
          "problem_id": "bug_003",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_004",
          "title": "Wrong variable name used",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_005",
          "title": "Wrong string method: split vs strip",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_006",
          "title": "Missing -1 in length calculation",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_007",
          "title": "Wrong list method: append vs extend",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_008",
          "title": "Comparing to None with == instead of is",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_009",
          "title": "Integer division instead of float division",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_010",
          "title": "Wrong list index: -1 vs 0",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_011",
          "title": "Wrong loop termination condition",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Wrong recursion base case value",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_013",
          "title": "Wrong sort order (missing reverse=True)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_014",
          "title": "Missing edge case for empty list",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.5625,
          "solved": false
        },
        {
          "problem_id": "bug_015",
          "title": "Wrong string format (% vs .format)",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.3,
          "solved": false
        },
        {
          "problem_id": "bug_016",
          "title": "Wrong dictionary key access",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.475,
          "solved": false
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong boolean logic (and vs or)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong range bounds (range(n) vs range(n+1))",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_019",
          "title": "Missing int() type conversion",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_020",
          "title": "Wrong list slicing ([1:] vs [:1])",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search off-by-one in mid update",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.5625,
          "solved": false
        },
        {
          "problem_id": "bug_022",
          "title": "Wrong DP state transition (Fibonacci)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_023",
          "title": "Wrong graph traversal: visited check in wrong place",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Missing memoization causing exponential recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Wrong two-pointer movement direction",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_026",
          "title": "Wrong greedy choice (min vs max)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_027",
          "title": "Off-by-one in sliding window boundary",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_028",
          "title": "Wrong merge logic in merge sort",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_029",
          "title": "Wrong heap operation (push vs pop)",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_030",
          "title": "Wrong backtracking: missing undo step",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        }
      ]
    },
    "oracle": {
      "baseline": "oracle",
      "total_problems": 30,
      "solved": 26,
      "solve_rate": 0.8667,
      "avg_reward": 0.9363,
      "avg_steps": 1.0,
      "avg_pass_rate": 0.9167,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in list index",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9957,
          "solved": true
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong comparison operator (> instead of >=)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9989,
          "solved": true
        },
        {
          "problem_id": "bug_003",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9791,
          "solved": true
        },
        {
          "problem_id": "bug_004",
          "title": "Wrong variable name used",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9985,
          "solved": true
        },
        {
          "problem_id": "bug_005",
          "title": "Wrong string method: split vs strip",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.4414,
          "solved": false
        },
        {
          "problem_id": "bug_006",
          "title": "Missing -1 in length calculation",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9981,
          "solved": true
        },
        {
          "problem_id": "bug_007",
          "title": "Wrong list method: append vs extend",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9954,
          "solved": true
        },
        {
          "problem_id": "bug_008",
          "title": "Comparing to None with == instead of is",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9952,
          "solved": true
        },
        {
          "problem_id": "bug_009",
          "title": "Integer division instead of float division",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.999,
          "solved": true
        },
        {
          "problem_id": "bug_010",
          "title": "Wrong list index: -1 vs 0",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9961,
          "solved": true
        },
        {
          "problem_id": "bug_011",
          "title": "Wrong loop termination condition",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9986,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Wrong recursion base case value",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9988,
          "solved": true
        },
        {
          "problem_id": "bug_013",
          "title": "Wrong sort order (missing reverse=True)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.986,
          "solved": true
        },
        {
          "problem_id": "bug_014",
          "title": "Missing edge case for empty list",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9684,
          "solved": true
        },
        {
          "problem_id": "bug_015",
          "title": "Wrong string format (% vs .format)",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.4363,
          "solved": false
        },
        {
          "problem_id": "bug_016",
          "title": "Wrong dictionary key access",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9848,
          "solved": true
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong boolean logic (and vs or)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9937,
          "solved": true
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong range bounds (range(n) vs range(n+1))",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9955,
          "solved": true
        },
        {
          "problem_id": "bug_019",
          "title": "Missing int() type conversion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9935,
          "solved": true
        },
        {
          "problem_id": "bug_020",
          "title": "Wrong list slicing ([1:] vs [:1])",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9971,
          "solved": true
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search off-by-one in mid update",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9985,
          "solved": true
        },
        {
          "problem_id": "bug_022",
          "title": "Wrong DP state transition (Fibonacci)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9994,
          "solved": true
        },
        {
          "problem_id": "bug_023",
          "title": "Wrong graph traversal: visited check in wrong place",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.969,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Missing memoization causing exponential recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9733,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Wrong two-pointer movement direction",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.6992,
          "solved": false
        },
        {
          "problem_id": "bug_026",
          "title": "Wrong greedy choice (min vs max)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9963,
          "solved": true
        },
        {
          "problem_id": "bug_027",
          "title": "Off-by-one in sliding window boundary",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.7115,
          "solved": false
        },
        {
          "problem_id": "bug_028",
          "title": "Wrong merge logic in merge sort",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9995,
          "solved": true
        },
        {
          "problem_id": "bug_029",
          "title": "Wrong heap operation (push vs pop)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9943,
          "solved": true
        },
        {
          "problem_id": "bug_030",
          "title": "Wrong backtracking: missing undo step",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9968,
          "solved": true
        }
      ]
    },
    "llm": {
      "baseline": "llm_llama-3.3-70b-versatile",
      "total_problems": 30,
      "solved": 27,
      "solve_rate": 0.9,
      "avg_reward": 1.1046,
      "avg_steps": 1.37,
      "avg_pass_rate": 0.9667,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in list index",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9647,
          "solved": true
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong comparison operator (> instead of >=)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9338,
          "solved": true
        },
        {
          "problem_id": "bug_003",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.98,
          "solved": true
        },
        {
          "problem_id": "bug_004",
          "title": "Wrong variable name used",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9977,
          "solved": true
        },
        {
          "problem_id": "bug_005",
          "title": "Wrong string method: split vs strip",
          "steps_taken": 3,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.56,
          "solved": true
        },
        {
          "problem_id": "bug_006",
          "title": "Missing -1 in length calculation",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9441,
          "solved": true
        },
        {
          "problem_id": "bug_007",
          "title": "Wrong list method: append vs extend",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9533,
          "solved": true
        },
        {
          "problem_id": "bug_008",
          "title": "Comparing to None with == instead of is",
          "steps_taken": 3,
          "best_pass_rate": 0.5,
          "cumulative_reward": 1.498,
          "solved": false
        },
        {
          "problem_id": "bug_009",
          "title": "Integer division instead of float division",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.942,
          "solved": true
        },
        {
          "problem_id": "bug_010",
          "title": "Wrong list index: -1 vs 0",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9605,
          "solved": true
        },
        {
          "problem_id": "bug_011",
          "title": "Wrong loop termination condition",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.94,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Wrong recursion base case value",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9459,
          "solved": true
        },
        {
          "problem_id": "bug_013",
          "title": "Wrong sort order (missing reverse=True)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9469,
          "solved": true
        },
        {
          "problem_id": "bug_014",
          "title": "Missing edge case for empty list",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9388,
          "solved": true
        },
        {
          "problem_id": "bug_015",
          "title": "Wrong string format (% vs .format)",
          "steps_taken": 2,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.409,
          "solved": true
        },
        {
          "problem_id": "bug_016",
          "title": "Wrong dictionary key access",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9539,
          "solved": true
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong boolean logic (and vs or)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.949,
          "solved": true
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong range bounds (range(n) vs range(n+1))",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9312,
          "solved": true
        },
        {
          "problem_id": "bug_019",
          "title": "Missing int() type conversion",
          "steps_taken": 2,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.3108,
          "solved": true
        },
        {
          "problem_id": "bug_020",
          "title": "Wrong list slicing ([1:] vs [:1])",
          "steps_taken": 2,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.4,
          "solved": true
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search off-by-one in mid update",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9853,
          "solved": true
        },
        {
          "problem_id": "bug_022",
          "title": "Wrong DP state transition (Fibonacci)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9898,
          "solved": true
        },
        {
          "problem_id": "bug_023",
          "title": "Wrong graph traversal: visited check in wrong place",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9804,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Missing memoization causing exponential recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9418,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Wrong two-pointer movement direction",
          "steps_taken": 3,
          "best_pass_rate": 0.75,
          "cumulative_reward": 1.9622,
          "solved": false
        },
        {
          "problem_id": "bug_026",
          "title": "Wrong greedy choice (min vs max)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9812,
          "solved": true
        },
        {
          "problem_id": "bug_027",
          "title": "Off-by-one in sliding window boundary",
          "steps_taken": 3,
          "best_pass_rate": 0.75,
          "cumulative_reward": 1.877,
          "solved": false
        },
        {
          "problem_id": "bug_028",
          "title": "Wrong merge logic in merge sort",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9856,
          "solved": true
        },
        {
          "problem_id": "bug_029",
          "title": "Wrong heap operation (push vs pop)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.994,
          "solved": true
        },
        {
          "problem_id": "bug_030",
          "title": "Wrong backtracking: missing undo step",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.982,
          "solved": true
        }
      ]
    }
  }
}