{
  "timestamp": "2026-04-25T10:07:45.034726",
  "baselines": {
    "no_change": {
      "baseline": "no_change",
      "total_problems": 30,
      "solved": 1,
      "solve_rate": 0.0333,
      "avg_reward": 0.5503,
      "avg_steps": 1.0,
      "avg_pass_rate": 0.3056,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in range",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong default mutable argument",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_003",
          "title": "Integer division instead of float",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_004",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_005",
          "title": "String concatenation type error",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_006",
          "title": "Wrong comparison operator",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.2,
          "solved": false
        },
        {
          "problem_id": "bug_007",
          "title": "Index out of range in list access",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_008",
          "title": "Infinite recursion \u2014 missing base case",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_009",
          "title": "Wrong variable in return",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_010",
          "title": "Dictionary KeyError",
          "steps_taken": 1,
          "best_pass_rate": 0.3333333333333333,
          "cumulative_reward": 0.5667,
          "solved": false
        },
        {
          "problem_id": "bug_011",
          "title": "Reversed sort order",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_012",
          "title": "Mutating input list unintentionally",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_013",
          "title": "Forgotten self in method",
          "steps_taken": 1,
          "best_pass_rate": 0.3333333333333333,
          "cumulative_reward": 0.5667,
          "solved": false
        },
        {
          "problem_id": "bug_014",
          "title": "Wrong boolean logic",
          "steps_taken": 1,
          "best_pass_rate": 0.3333333333333333,
          "cumulative_reward": 0.5667,
          "solved": false
        },
        {
          "problem_id": "bug_015",
          "title": "Scope issue with global variable",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_016",
          "title": "Shallow copy issue",
          "steps_taken": 1,
          "best_pass_rate": 0.5,
          "cumulative_reward": 0.625,
          "solved": false
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong string method",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong exception handling",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_019",
          "title": "Fibonacci off-by-one",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.0,
          "solved": true
        },
        {
          "problem_id": "bug_020",
          "title": "List comprehension filter wrong",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search wrong midpoint update",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.3,
          "solved": false
        },
        {
          "problem_id": "bug_022",
          "title": "Reversed dictionary comprehension",
          "steps_taken": 1,
          "best_pass_rate": 0.3333333333333333,
          "cumulative_reward": 0.5667,
          "solved": false
        },
        {
          "problem_id": "bug_023",
          "title": "String reverse off by one",
          "steps_taken": 1,
          "best_pass_rate": 0.3333333333333333,
          "cumulative_reward": 0.5667,
          "solved": false
        },
        {
          "problem_id": "bug_024",
          "title": "Wrong accumulator init for max",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_025",
          "title": "Nested loop wrong variable",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_026",
          "title": "Palindrome check wrong slice",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_027",
          "title": "Power function wrong operator",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_028",
          "title": "GCD wrong recursion",
          "steps_taken": 1,
          "best_pass_rate": 0.0,
          "cumulative_reward": 0.45,
          "solved": false
        },
        {
          "problem_id": "bug_029",
          "title": "Zip truncation issue",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        },
        {
          "problem_id": "bug_030",
          "title": "Matrix transpose wrong indices",
          "steps_taken": 1,
          "best_pass_rate": 0.6666666666666666,
          "cumulative_reward": 0.6833,
          "solved": false
        }
      ]
    },
    "oracle": {
      "baseline": "oracle",
      "total_problems": 30,
      "solved": 30,
      "solve_rate": 1.0,
      "avg_reward": 0.9904,
      "avg_steps": 1.0,
      "avg_pass_rate": 1.0,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in range",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9978,
          "solved": true
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong default mutable argument",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9751,
          "solved": true
        },
        {
          "problem_id": "bug_003",
          "title": "Integer division instead of float",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.999,
          "solved": true
        },
        {
          "problem_id": "bug_004",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9786,
          "solved": true
        },
        {
          "problem_id": "bug_005",
          "title": "String concatenation type error",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9971,
          "solved": true
        },
        {
          "problem_id": "bug_006",
          "title": "Wrong comparison operator",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9986,
          "solved": true
        },
        {
          "problem_id": "bug_007",
          "title": "Index out of range in list access",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9959,
          "solved": true
        },
        {
          "problem_id": "bug_008",
          "title": "Infinite recursion \u2014 missing base case",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9754,
          "solved": true
        },
        {
          "problem_id": "bug_009",
          "title": "Wrong variable in return",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9985,
          "solved": true
        },
        {
          "problem_id": "bug_010",
          "title": "Dictionary KeyError",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9877,
          "solved": true
        },
        {
          "problem_id": "bug_011",
          "title": "Reversed sort order",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9873,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Mutating input list unintentionally",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9611,
          "solved": true
        },
        {
          "problem_id": "bug_013",
          "title": "Forgotten self in method",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9984,
          "solved": true
        },
        {
          "problem_id": "bug_014",
          "title": "Wrong boolean logic",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9954,
          "solved": true
        },
        {
          "problem_id": "bug_015",
          "title": "Scope issue with global variable",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9831,
          "solved": true
        },
        {
          "problem_id": "bug_016",
          "title": "Shallow copy issue",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9977,
          "solved": true
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong string method",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9966,
          "solved": true
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong exception handling",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9926,
          "solved": true
        },
        {
          "problem_id": "bug_019",
          "title": "Fibonacci off-by-one",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9981,
          "solved": true
        },
        {
          "problem_id": "bug_020",
          "title": "List comprehension filter wrong",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9984,
          "solved": true
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search wrong midpoint update",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9985,
          "solved": true
        },
        {
          "problem_id": "bug_022",
          "title": "Reversed dictionary comprehension",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9966,
          "solved": true
        },
        {
          "problem_id": "bug_023",
          "title": "String reverse off by one",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9921,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Wrong accumulator init for max",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9964,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Nested loop wrong variable",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9974,
          "solved": true
        },
        {
          "problem_id": "bug_026",
          "title": "Palindrome check wrong slice",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.983,
          "solved": true
        },
        {
          "problem_id": "bug_027",
          "title": "Power function wrong operator",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9966,
          "solved": true
        },
        {
          "problem_id": "bug_028",
          "title": "GCD wrong recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9986,
          "solved": true
        },
        {
          "problem_id": "bug_029",
          "title": "Zip truncation issue",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9551,
          "solved": true
        },
        {
          "problem_id": "bug_030",
          "title": "Matrix transpose wrong indices",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9858,
          "solved": true
        }
      ]
    },
    "llm": {
      "baseline": "llm_llama-3.3-70b-versatile",
      "total_problems": 30,
      "solved": 29,
      "solve_rate": 0.9667,
      "avg_reward": 1.0346,
      "avg_steps": 1.17,
      "avg_pass_rate": 0.9833,
      "results": [
        {
          "problem_id": "bug_001",
          "title": "Off-by-one in range",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9963,
          "solved": true
        },
        {
          "problem_id": "bug_002",
          "title": "Wrong default mutable argument",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9751,
          "solved": true
        },
        {
          "problem_id": "bug_003",
          "title": "Integer division instead of float",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9548,
          "solved": true
        },
        {
          "problem_id": "bug_004",
          "title": "Missing return statement",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9786,
          "solved": true
        },
        {
          "problem_id": "bug_005",
          "title": "String concatenation type error",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9971,
          "solved": true
        },
        {
          "problem_id": "bug_006",
          "title": "Wrong comparison operator",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9986,
          "solved": true
        },
        {
          "problem_id": "bug_007",
          "title": "Index out of range in list access",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9448,
          "solved": true
        },
        {
          "problem_id": "bug_008",
          "title": "Infinite recursion \u2014 missing base case",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9473,
          "solved": true
        },
        {
          "problem_id": "bug_009",
          "title": "Wrong variable in return",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9985,
          "solved": true
        },
        {
          "problem_id": "bug_010",
          "title": "Dictionary KeyError",
          "steps_taken": 2,
          "best_pass_rate": 1.0,
          "cumulative_reward": 1.4187,
          "solved": true
        },
        {
          "problem_id": "bug_011",
          "title": "Reversed sort order",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9568,
          "solved": true
        },
        {
          "problem_id": "bug_012",
          "title": "Mutating input list unintentionally",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.968,
          "solved": true
        },
        {
          "problem_id": "bug_013",
          "title": "Forgotten self in method",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9886,
          "solved": true
        },
        {
          "problem_id": "bug_014",
          "title": "Wrong boolean logic",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9954,
          "solved": true
        },
        {
          "problem_id": "bug_015",
          "title": "Scope issue with global variable",
          "steps_taken": 3,
          "best_pass_rate": 0.5,
          "cumulative_reward": 1.1811,
          "solved": false
        },
        {
          "problem_id": "bug_016",
          "title": "Shallow copy issue",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9656,
          "solved": true
        },
        {
          "problem_id": "bug_017",
          "title": "Wrong string method",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9966,
          "solved": true
        },
        {
          "problem_id": "bug_018",
          "title": "Wrong exception handling",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9897,
          "solved": true
        },
        {
          "problem_id": "bug_019",
          "title": "Fibonacci off-by-one",
          "steps_taken": 3,
          "best_pass_rate": 1.0,
          "cumulative_reward": 2.0001,
          "solved": true
        },
        {
          "problem_id": "bug_020",
          "title": "List comprehension filter wrong",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9984,
          "solved": true
        },
        {
          "problem_id": "bug_021",
          "title": "Binary search wrong midpoint update",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9901,
          "solved": true
        },
        {
          "problem_id": "bug_022",
          "title": "Reversed dictionary comprehension",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9651,
          "solved": true
        },
        {
          "problem_id": "bug_023",
          "title": "String reverse off by one",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9921,
          "solved": true
        },
        {
          "problem_id": "bug_024",
          "title": "Wrong accumulator init for max",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9719,
          "solved": true
        },
        {
          "problem_id": "bug_025",
          "title": "Nested loop wrong variable",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9883,
          "solved": true
        },
        {
          "problem_id": "bug_026",
          "title": "Palindrome check wrong slice",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.983,
          "solved": true
        },
        {
          "problem_id": "bug_027",
          "title": "Power function wrong operator",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9966,
          "solved": true
        },
        {
          "problem_id": "bug_028",
          "title": "GCD wrong recursion",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9567,
          "solved": true
        },
        {
          "problem_id": "bug_029",
          "title": "Zip truncation issue",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9768,
          "solved": true
        },
        {
          "problem_id": "bug_030",
          "title": "Matrix transpose wrong indices",
          "steps_taken": 1,
          "best_pass_rate": 1.0,
          "cumulative_reward": 0.9677,
          "solved": true
        }
      ]
    }
  }
}