{
  "timestamp": "2026-04-25T10:23:42.599050",
  "baselines": {
    "no_change": {
      "baseline": "no_change",
      "total_problems": 30,
      "solved": 4,
      "solve_rate": 0.1333,
      "avg_reward": 0.5887,
      "avg_steps": 1.0,
      "avg_pass_rate": 0.3917,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in list index",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.3,
          "solved": false
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong comparison operator (> instead of >=)",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.7125,
          "solved": false
        },
        {
          "problem_id": "bug_003",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_004",
          "title": "Wrong variable name used",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_005",
          "title": "Wrong string method: split vs strip",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_006",
          "title": "Missing -1 in length calculation",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_007",
          "title": "Wrong list method: append vs extend",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_008",
          "title": "Comparing to None with == instead of is",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_009",
          "title": "Integer division instead of float division",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_010",
          "title": "Wrong list index: -1 vs 0",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_011",
          "title": "Wrong loop termination condition",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Wrong recursion base case value",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_013",
          "title": "Wrong sort order (missing reverse=True)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_014",
          "title": "Missing edge case for empty list",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.5625,
          "solved": false
        },
        {
          "problem_id": "bug_015",
          "title": "Wrong string format (% vs .format)",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.3,
          "solved": false
        },
        {
          "problem_id": "bug_016",
          "title": "Wrong dictionary key access",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.475,
          "solved": false
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong boolean logic (and vs or)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong range bounds (range(n) vs range(n+1))",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_019",
          "title": "Missing int() type conversion",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_020",
          "title": "Wrong list slicing ([1:] vs [:1])",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search off-by-one in mid update",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.5625,
          "solved": false
        },
        {
          "problem_id": "bug_022",
          "title": "Wrong DP state transition (Fibonacci)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_023",
          "title": "Wrong graph traversal: visited check in wrong place",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Missing memoization causing exponential recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Wrong two-pointer movement direction",
          "steps_taken": 1,
          "best_pass_rate": 0.75,
          "cumulative_reward": 0.7125,
          "solved": false
        },
        {
          "problem_id": "bug_026",
          "title": "Wrong greedy choice (min vs max)",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_027",
          "title": "Off-by-one in sliding window boundary",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_028",
          "title": "Wrong merge logic in merge sort",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_029",
          "title": "Wrong heap operation (push vs pop)",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        },
        {
          "problem_id": "bug_030",
          "title": "Wrong backtracking: missing undo step",
          "steps_taken": 1,
          "best_pass_rate": 0.25,
          "cumulative_reward": 0.5375,
          "solved": false
        }
      ]
    },
    "oracle": {
      "baseline": "oracle",
      "total_problems": 30,
      "solved": 30,
      "solve_rate": 1.0,
      "avg_reward": 0.9921,
      "avg_steps": 1.0,
      "avg_pass_rate": 1.0,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in list index",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9957,
          "solved": true
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong comparison operator (> instead of >=)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9989,
          "solved": true
        },
        {
          "problem_id": "bug_003",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9791,
          "solved": true
        },
        {
          "problem_id": "bug_004",
          "title": "Wrong variable name used",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9985,
          "solved": true
        },
        {
          "problem_id": "bug_005",
          "title": "Wrong string method: split vs strip",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9914,
          "solved": true
        },
        {
          "problem_id": "bug_006",
          "title": "Missing -1 in length calculation",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9981,
          "solved": true
        },
        {
          "problem_id": "bug_007",
          "title": "Wrong list method: append vs extend",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9954,
          "solved": true
        },
        {
          "problem_id": "bug_008",
          "title": "Comparing to None with == instead of is",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9952,
          "solved": true
        },
        {
          "problem_id": "bug_009",
          "title": "Integer division instead of float division",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.999,
          "solved": true
        },
        {
          "problem_id": "bug_010",
          "title": "Wrong list index: -1 vs 0",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9961,
          "solved": true
        },
        {
          "problem_id": "bug_011",
          "title": "Wrong loop termination condition",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9986,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Wrong recursion base case value",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9988,
          "solved": true
        },
        {
          "problem_id": "bug_013",
          "title": "Wrong sort order (missing reverse=True)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.986,
          "solved": true
        },
        {
          "problem_id": "bug_014",
          "title": "Missing edge case for empty list",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9684,
          "solved": true
        },
        {
          "problem_id": "bug_015",
          "title": "Wrong string format (% vs .format)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9863,
          "solved": true
        },
        {
          "problem_id": "bug_016",
          "title": "Wrong dictionary key access",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9848,
          "solved": true
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong boolean logic (and vs or)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9937,
          "solved": true
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong range bounds (range(n) vs range(n+1))",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9955,
          "solved": true
        },
        {
          "problem_id": "bug_019",
          "title": "Missing int() type conversion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9935,
          "solved": true
        },
        {
          "problem_id": "bug_020",
          "title": "Wrong list slicing ([1:] vs [:1])",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9971,
          "solved": true
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search off-by-one in mid update",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9985,
          "solved": true
        },
        {
          "problem_id": "bug_022",
          "title": "Wrong DP state transition (Fibonacci)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9994,
          "solved": true
        },
        {
          "problem_id": "bug_023",
          "title": "Wrong graph traversal: visited check in wrong place",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.969,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Missing memoization causing exponential recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9733,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Wrong two-pointer movement direction",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9867,
          "solved": true
        },
        {
          "problem_id": "bug_026",
          "title": "Wrong greedy choice (min vs max)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9963,
          "solved": true
        },
        {
          "problem_id": "bug_027",
          "title": "Off-by-one in sliding window boundary",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.999,
          "solved": true
        },
        {
          "problem_id": "bug_028",
          "title": "Wrong merge logic in merge sort",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9995,
          "solved": true
        },
        {
          "problem_id": "bug_029",
          "title": "Wrong heap operation (push vs pop)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9943,
          "solved": true
        },
        {
          "problem_id": "bug_030",
          "title": "Wrong backtracking: missing undo step",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9968,
          "solved": true
        }
      ]
    },
    "llm": {
      "baseline": "llm_llama-3.3-70b-versatile",
      "total_problems": 30,
      "solved": 29,
      "solve_rate": 0.9667,
      "avg_reward": 1.0056,
      "avg_steps": 1.13,
      "avg_pass_rate": 0.9833,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in list index",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9391,
          "solved": true
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong comparison operator (> instead of >=)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9682,
          "solved": true
        },
        {
          "problem_id": "bug_003",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.98,
          "solved": true
        },
        {
          "problem_id": "bug_004",
          "title": "Wrong variable name used",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9977,
          "solved": true
        },
        {
          "problem_id": "bug_005",
          "title": "Wrong string method: split vs strip",
          "steps_taken": 2,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.2898,
          "solved": true
        },
        {
          "problem_id": "bug_006",
          "title": "Missing -1 in length calculation",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9448,
          "solved": true
        },
        {
          "problem_id": "bug_007",
          "title": "Wrong list method: append vs extend",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9537,
          "solved": true
        },
        {
          "problem_id": "bug_008",
          "title": "Comparing to None with == instead of is",
          "steps_taken": 3,
          "best_pass_rate": 0.5,
          "cumulative_reward": 1.5463,
          "solved": false
        },
        {
          "problem_id": "bug_009",
          "title": "Integer division instead of float division",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9295,
          "solved": true
        },
        {
          "problem_id": "bug_010",
          "title": "Wrong list index: -1 vs 0",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9605,
          "solved": true
        },
        {
          "problem_id": "bug_011",
          "title": "Wrong loop termination condition",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.941,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Wrong recursion base case value",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9755,
          "solved": true
        },
        {
          "problem_id": "bug_013",
          "title": "Wrong sort order (missing reverse=True)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9519,
          "solved": true
        },
        {
          "problem_id": "bug_014",
          "title": "Missing edge case for empty list",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9388,
          "solved": true
        },
        {
          "problem_id": "bug_015",
          "title": "Wrong string format (% vs .format)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9968,
          "solved": true
        },
        {
          "problem_id": "bug_016",
          "title": "Wrong dictionary key access",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9231,
          "solved": true
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong boolean logic (and vs or)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.949,
          "solved": true
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong range bounds (range(n) vs range(n+1))",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9402,
          "solved": true
        },
        {
          "problem_id": "bug_019",
          "title": "Missing int() type conversion",
          "steps_taken": 2,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.3067,
          "solved": true
        },
        {
          "problem_id": "bug_020",
          "title": "Wrong list slicing ([1:] vs [:1])",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9561,
          "solved": true
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search off-by-one in mid update",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9931,
          "solved": true
        },
        {
          "problem_id": "bug_022",
          "title": "Wrong DP state transition (Fibonacci)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9888,
          "solved": true
        },
        {
          "problem_id": "bug_023",
          "title": "Wrong graph traversal: visited check in wrong place",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9794,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Missing memoization causing exponential recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9736,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Wrong two-pointer movement direction",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.974,
          "solved": true
        },
        {
          "problem_id": "bug_026",
          "title": "Wrong greedy choice (min vs max)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9812,
          "solved": true
        },
        {
          "problem_id": "bug_027",
          "title": "Off-by-one in sliding window boundary",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9908,
          "solved": true
        },
        {
          "problem_id": "bug_028",
          "title": "Wrong merge logic in merge sort",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9877,
          "solved": true
        },
        {
          "problem_id": "bug_029",
          "title": "Wrong heap operation (push vs pop)",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.932,
          "solved": true
        },
        {
          "problem_id": "bug_030",
          "title": "Wrong backtracking: missing undo step",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9775,
          "solved": true
        }
      ]
    }
  }
}