computation_comparison_report.json

{
  "scope": 815,
  "brackets": {
    "computation_comparison": {
      "scope": 130,
      "percent": 15.95
    },
    "other": {
      "scope": 685,
      "percent": 84.05
    }
  },
  "runs": [
    {
      "slot": "c1",
      "label": "c1 Closed-book",
      "score_mode": "mean_judge_score",
      "brackets": {
        "computation_comparison": {
          "scope": 130,
          "answered": 130,
          "correct": 0,
          "score_pct": 0.0,
          "perfect_pct": 0.0,
          "coverage_pct": 100.0
        },
        "other": {
          "scope": 685,
          "answered": 685,
          "correct": 0,
          "score_pct": 5.05,
          "perfect_pct": 0.0,
          "coverage_pct": 100.0
        }
      }
    },
    {
      "slot": "c2",
      "label": "c2 With-docs",
      "score_mode": "mean_judge_score",
      "brackets": {
        "computation_comparison": {
          "scope": 130,
          "answered": 130,
          "correct": 61,
          "score_pct": 50.1,
          "perfect_pct": 46.92,
          "coverage_pct": 100.0
        },
        "other": {
          "scope": 685,
          "answered": 685,
          "correct": 324,
          "score_pct": 71.35,
          "perfect_pct": 47.3,
          "coverage_pct": 100.0
        }
      }
    },
    {
      "slot": "c6",
      "label": "c6 Agentic-DCI",
      "score_mode": "mean_judge_score",
      "brackets": {
        "computation_comparison": {
          "scope": 130,
          "answered": 126,
          "correct": 90,
          "score_pct": 79.5,
          "perfect_pct": 69.23,
          "coverage_pct": 96.92
        },
        "other": {
          "scope": 685,
          "answered": 666,
          "correct": 296,
          "score_pct": 74.91,
          "perfect_pct": 43.21,
          "coverage_pct": 97.23
        }
      }
    },
    {
      "slot": "naive",
      "label": "Naive-search",
      "score_mode": "mean_judge_score",
      "brackets": {
        "computation_comparison": {
          "scope": 130,
          "answered": 26,
          "correct": 21,
          "score_pct": 18.21,
          "perfect_pct": 16.15,
          "coverage_pct": 20.0
        },
        "other": {
          "scope": 685,
          "answered": 521,
          "correct": 157,
          "score_pct": 51.22,
          "perfect_pct": 22.92,
          "coverage_pct": 76.06
        }
      }
    },
    {
      "slot": "e2e_raw",
      "label": "E2E raw",
      "score_mode": "mean_judge_score",
      "brackets": {
        "computation_comparison": {
          "scope": 130,
          "answered": 125,
          "correct": 75,
          "score_pct": 71.64,
          "perfect_pct": 57.69,
          "coverage_pct": 96.15
        },
        "other": {
          "scope": 685,
          "answered": 673,
          "correct": 209,
          "score_pct": 67.16,
          "perfect_pct": 30.51,
          "coverage_pct": 98.25
        }
      }
    },
    {
      "slot": "e2e_combined",
      "label": "E2E combined",
      "score_mode": "mean_judge_score",
      "brackets": {
        "computation_comparison": {
          "scope": 130,
          "answered": 130,
          "correct": 80,
          "score_pct": 72.83,
          "perfect_pct": 61.54,
          "coverage_pct": 100.0
        },
        "other": {
          "scope": 685,
          "answered": 683,
          "correct": 256,
          "score_pct": 74.12,
          "perfect_pct": 37.37,
          "coverage_pct": 99.71
        }
      }
    },
    {
      "slot": "e2e_rawtext",
      "label": "E2E v3 + rawtext",
      "score_mode": "mean_judge_score",
      "brackets": {
        "computation_comparison": {
          "scope": 130,
          "answered": 129,
          "correct": 88,
          "score_pct": 78.05,
          "perfect_pct": 67.69,
          "coverage_pct": 99.23
        },
        "other": {
          "scope": 685,
          "answered": 683,
          "correct": 290,
          "score_pct": 77.55,
          "perfect_pct": 42.34,
          "coverage_pct": 99.71
        }
      }
    }
  ]
}