artifacts/baseline_scores.json

{
  "mode": "offline",
  "model_name": "Qwen/Qwen2.5-72B-Instruct",
  "api_base_url": "https://router.huggingface.co/v1",
  "max_steps": 8,
  "policy_mode": "imp",
  "trajectories": 3,
  "average_score": 0.66,
  "success_rate": 0.333,
  "results": [
    {
      "task_id": "easy-command-typo",
      "difficulty": "easy",
      "success": true,
      "steps": 7,
      "score": 0.672,
      "rewards": [
        0.302,
        0.362,
        0.57,
        0.63,
        0.84,
        0.999,
        0.999
      ],
      "start_line": "[START] task=easy-command-typo env=cicd_debugger_env model=Qwen/Qwen2.5-72B-Instruct",
      "end_line": "[END] success=true steps=7 score=0.672 rewards=0.302,0.362,0.570,0.630,0.840,0.999,0.999"
    },
    {
      "task_id": "medium-python-version",
      "difficulty": "medium",
      "success": false,
      "steps": 8,
      "score": 0.68,
      "rewards": [
        0.482,
        0.542,
        0.582,
        0.792,
        0.999,
        0.999,
        0.521,
        0.521
      ],
      "start_line": "[START] task=medium-python-version env=cicd_debugger_env model=Qwen/Qwen2.5-72B-Instruct",
      "end_line": "[END] success=false steps=8 score=0.680 rewards=0.482,0.542,0.582,0.792,0.999,0.999,0.521,0.521"
    },
    {
      "task_id": "hard-needs-order",
      "difficulty": "hard",
      "success": false,
      "steps": 8,
      "score": 0.629,
      "rewards": [
        0.48,
        0.54,
        0.522,
        0.568,
        0.458,
        0.628,
        0.838,
        0.999
      ],
      "start_line": "[START] task=hard-needs-order env=cicd_debugger_env model=Qwen/Qwen2.5-72B-Instruct",
      "end_line": "[END] success=false steps=8 score=0.629 rewards=0.480,0.540,0.522,0.568,0.458,0.628,0.838,0.999"
    }
  ]
}