live_model_comparison.json

{
  "generated_at_utc": "2026-04-08T23:04:55.197968+00:00",
  "models": [
    "gpt-3.5-turbo",
    "gpt-4o",
    "gpt-5.4"
  ],
  "cases": [
    "CASE-A-001",
    "CASE-A-002",
    "CASE-A-003",
    "CASE-A-004",
    "CASE-B-001",
    "CASE-B-002",
    "CASE-B-003",
    "CASE-B-004",
    "CASE-B-005",
    "CASE-C-001",
    "CASE-C-002",
    "CASE-C-003",
    "CASE-C-004",
    "CASE-D-001",
    "CASE-D-002",
    "CASE-D-003",
    "CASE-D-004",
    "CASE-D-005",
    "CASE-D-006",
    "CASE-E-001",
    "CASE-E-002"
  ],
  "case_count": 21,
  "pass_threshold": 0.85,
  "runtime_context": {
    "python_version": "3.12.7",
    "python_executable": "/opt/anaconda3/bin/python",
    "api_base_url": "https://api.openai.com/v1",
    "env_url": "http://127.0.0.1:8000"
  },
  "capability_order": {
    "ordered_models": [
      "gpt-3.5-turbo",
      "gpt-4o",
      "gpt-5.4"
    ],
    "monotonic_by_capability": true,
    "pairwise": [
      {
        "weaker_model": "gpt-3.5-turbo",
        "stronger_model": "gpt-4o",
        "weaker_tier": "standard",
        "stronger_tier": "strong",
        "average_score_gap": 0.1654,
        "success_rate_gap": 0.4286,
        "ordering_ok": true
      },
      {
        "weaker_model": "gpt-4o",
        "stronger_model": "gpt-5.4",
        "weaker_tier": "strong",
        "stronger_tier": "elite",
        "average_score_gap": 0.0642,
        "success_rate_gap": 0.1905,
        "ordering_ok": true
      }
    ],
    "violations": []
  },
  "results": [
    {
      "model": "gpt-3.5-turbo",
      "model_profile": {
        "capability_score": 3.2,
        "decision_token_budget": 1024,
        "intervention_budget_bonus": 0,
        "investigation_budget_bonus": 0,
        "model_name": "gpt-3.5-turbo",
        "plan_mode": "llm",
        "planning_token_budget": 384,
        "repair_level": "none",
        "tier": "standard"
      },
      "average_score": 0.7009,
      "success_rate": 0.381,
      "min_score": 0.01,
      "max_score": 0.99,
      "api_calls": 63,
      "debug_artifact_dir": "live_model_comparison_debug/gpt-3.5-turbo",
      "failed_cases": [
        "CASE-B-005",
        "CASE-C-001",
        "CASE-C-002",
        "CASE-C-003",
        "CASE-C-004",
        "CASE-D-001",
        "CASE-D-002",
        "CASE-D-003",
        "CASE-D-004",
        "CASE-D-005",
        "CASE-D-006",
        "CASE-E-001",
        "CASE-E-002"
      ],
      "case_scores": {
        "CASE-A-001": 0.9882,
        "CASE-A-002": 0.9882,
        "CASE-A-003": 0.9402,
        "CASE-A-004": 0.9403,
        "CASE-B-001": 0.8797,
        "CASE-B-002": 0.8708,
        "CASE-B-003": 0.99,
        "CASE-B-004": 0.8558,
        "CASE-B-005": 0.3919,
        "CASE-C-001": 0.01,
        "CASE-C-002": 0.8078,
        "CASE-C-003": 0.1357,
        "CASE-C-004": 0.8044,
        "CASE-D-001": 0.6294,
        "CASE-D-002": 0.7779,
        "CASE-D-003": 0.6361,
        "CASE-D-004": 0.5932,
        "CASE-D-005": 0.7809,
        "CASE-D-006": 0.7641,
        "CASE-E-001": 0.7194,
        "CASE-E-002": 0.2142
      }
    },
    {
      "model": "gpt-4o",
      "model_profile": {
        "capability_score": 4.6,
        "decision_token_budget": 1280,
        "intervention_budget_bonus": 1,
        "investigation_budget_bonus": 1,
        "model_name": "gpt-4o",
        "plan_mode": "hybrid",
        "planning_token_budget": 512,
        "repair_level": "partial",
        "tier": "strong"
      },
      "average_score": 0.8663,
      "success_rate": 0.8095,
      "min_score": 0.4252,
      "max_score": 0.99,
      "api_calls": 65,
      "debug_artifact_dir": "live_model_comparison_debug/gpt-4o",
      "failed_cases": [
        "CASE-B-002",
        "CASE-B-004",
        "CASE-C-002",
        "CASE-E-001"
      ],
      "case_scores": {
        "CASE-A-001": 0.9882,
        "CASE-A-002": 0.9882,
        "CASE-A-003": 0.9402,
        "CASE-A-004": 0.9403,
        "CASE-B-001": 0.9488,
        "CASE-B-002": 0.579,
        "CASE-B-003": 0.99,
        "CASE-B-004": 0.5551,
        "CASE-B-005": 0.99,
        "CASE-C-001": 0.9221,
        "CASE-C-002": 0.4252,
        "CASE-C-003": 0.9207,
        "CASE-C-004": 0.8952,
        "CASE-D-001": 0.9213,
        "CASE-D-002": 0.8762,
        "CASE-D-003": 0.91,
        "CASE-D-004": 0.8911,
        "CASE-D-005": 0.8909,
        "CASE-D-006": 0.8762,
        "CASE-E-001": 0.84,
        "CASE-E-002": 0.903
      }
    },
    {
      "model": "gpt-5.4",
      "model_profile": {
        "capability_score": 5.4,
        "decision_token_budget": 1536,
        "intervention_budget_bonus": 2,
        "investigation_budget_bonus": 2,
        "model_name": "gpt-5.4",
        "plan_mode": "llm",
        "planning_token_budget": 640,
        "repair_level": "partial",
        "tier": "elite"
      },
      "average_score": 0.9305,
      "success_rate": 1.0,
      "min_score": 0.8752,
      "max_score": 0.99,
      "api_calls": 64,
      "debug_artifact_dir": "live_model_comparison_debug/gpt-5.4",
      "failed_cases": [],
      "case_scores": {
        "CASE-A-001": 0.9882,
        "CASE-A-002": 0.9882,
        "CASE-A-003": 0.9402,
        "CASE-A-004": 0.9403,
        "CASE-B-001": 0.9504,
        "CASE-B-002": 0.9092,
        "CASE-B-003": 0.99,
        "CASE-B-004": 0.8839,
        "CASE-B-005": 0.99,
        "CASE-C-001": 0.9216,
        "CASE-C-002": 0.9171,
        "CASE-C-003": 0.9202,
        "CASE-C-004": 0.9046,
        "CASE-D-001": 0.9176,
        "CASE-D-002": 0.8752,
        "CASE-D-003": 0.9359,
        "CASE-D-004": 0.9188,
        "CASE-D-005": 0.8854,
        "CASE-D-006": 0.8759,
        "CASE-E-001": 0.9618,
        "CASE-E-002": 0.9258
      }
    }
  ]
}