{
  "engine": "matilda-jev-di62p43-fp4",
  "edition": "0.2.1",
  "generated_utc": "2026-10-06T16:58:26+00:00",
  "suite": {
    "edition": "release-v2.1",
    "requests": 120340,
    "scoreable": 119898,
    "excluded": 442,
    "added_requests": 30419,
    "benchmarks": 44,
    "rows_sha256": "b2b56d6fb636837ca469e689087bdbf373dda8de7638aa2da6793e6eda0792d5",
    "added_sha256": "7429f3c9cdddb772c1cfc42bb2a45e8516b0032152b746e6929f1c8b52f4ce89"
  },
  "completed": 150317,
  "complete": true,
  "counts": {
    "ok": 150317
  },
  "latency_ms": {
    "median": 71.3,
    "p95": 408.4,
    "mean": 137.5
  },
  "decision_index": 61.77,
  "raw_index": 70.9,
  "scores": {
    "balanced_skill": 61.77,
    "balanced_raw": 70.9,
    "breadth_skill": 60.48
  },
  "areas": [
    {
      "id": "knowledge",
      "label": "Knowledge & Reasoning",
      "raw": 0.5747,
      "skill": 0.4645,
      "coverage": 1.0,
      "n": 10,
      "benchmarks": [
        25,
        30,
        31,
        32,
        33,
        43,
        44,
        45,
        57,
        58
      ]
    },
    {
      "id": "language",
      "label": "Language Understanding",
      "raw": 0.7864,
      "skill": 0.6924,
      "coverage": 1.0,
      "n": 10,
      "benchmarks": [
        11,
        12,
        28,
        29,
        38,
        39,
        40,
        41,
        42,
        59
      ]
    },
    {
      "id": "retrieval",
      "label": "Retrieval & Classification",
      "raw": 0.7403,
      "skill": 0.6441,
      "coverage": 1.0,
      "n": 6,
      "benchmarks": [
        4,
        5,
        36,
        37,
        56,
        61
      ]
    },
    {
      "id": "tools",
      "label": "Tools & Automation",
      "raw": 0.8286,
      "skill": 0.8076,
      "coverage": 1.0,
      "n": 5,
      "benchmarks": [
        1,
        2,
        3,
        9,
        62
      ]
    },
    {
      "id": "arts",
      "label": "Arts & Human Taste",
      "raw": 0.5744,
      "skill": 0.4205,
      "coverage": 1.0,
      "n": 7,
      "benchmarks": [
        20,
        21,
        22,
        23,
        48,
        50,
        64
      ]
    }
  ],
  "index_benchmarks": {
    "1": {
      "raw": 0.9669,
      "skill": 0.9554,
      "coverage": 1.0,
      "random": 0.2592,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "2": {
      "raw": 0.676,
      "skill": 0.6258,
      "coverage": 1.0,
      "random": 0.1341,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "3": {
      "raw": 0.8465,
      "skill": 0.8435,
      "coverage": 1.0,
      "random": 0.0189,
      "rule": "chance",
      "in_index": true
    },
    "4": {
      "raw": 0.8953,
      "skill": 0.894,
      "coverage": 1.0,
      "random": 0.0127,
      "rule": "chance",
      "in_index": true
    },
    "5": {
      "raw": 0.912,
      "skill": 0.9115,
      "coverage": 1.0,
      "random": 0.006,
      "rule": "chance",
      "in_index": true
    },
    "9": {
      "raw": 0.7614,
      "skill": 0.7614,
      "coverage": 1.0,
      "random": 0.0,
      "rule": "chance",
      "in_index": true
    },
    "11": {
      "raw": 0.8323,
      "skill": 0.7574,
      "coverage": 1.0,
      "random": 0.3085,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "12": {
      "raw": 0.7372,
      "skill": 0.6064,
      "coverage": 1.0,
      "random": 0.3324,
      "rule": "chance",
      "in_index": true
    },
    "20": {
      "raw": 0.9467,
      "skill": 0.8934,
      "coverage": 1.0,
      "random": 0.5,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "21": {
      "raw": 0.6191,
      "skill": 0.2382,
      "coverage": 1.0,
      "random": 0.5,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "22": {
      "raw": 0.4118,
      "skill": 0.4072,
      "coverage": 1.0,
      "random": 0.0078,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "23": {
      "raw": 0.6553,
      "skill": 0.3106,
      "coverage": 1.0,
      "random": 0.5,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "25": {
      "raw": 0.4847,
      "skill": 0.3129,
      "coverage": 1.0,
      "random": 0.25,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "28": {
      "raw": 0.8603,
      "skill": 0.7206,
      "coverage": 1.0,
      "random": 0.5,
      "rule": "chance",
      "in_index": true
    },
    "29": {
      "raw": 0.952,
      "skill": 0.936,
      "coverage": 1.0,
      "random": 0.25,
      "rule": "chance",
      "in_index": true
    },
    "30": {
      "raw": 0.7953,
      "skill": 0.756,
      "coverage": 1.0,
      "random": 0.25,
      "rule": "track",
      "in_index": true,
      "tracks": [
        {
          "track": "GSM8K-4choice",
          "score": 0.8514,
          "headline": false
        },
        {
          "track": "GSM8K-10choice",
          "score": 0.7392,
          "headline": false
        }
      ]
    },
    "31": {
      "raw": 0.207,
      "skill": 0.1363,
      "coverage": 1.0,
      "random": 0.0819,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "32": {
      "raw": 0.6742,
      "skill": 0.482,
      "coverage": 1.0,
      "random": 0.371,
      "rule": "chance",
      "in_index": true
    },
    "33": {
      "raw": 0.2558,
      "skill": 0.2459,
      "coverage": 1.0,
      "random": 0.0131,
      "rule": "chance",
      "in_index": true
    },
    "36": {
      "raw": 0.4851,
      "skill": 0.4175,
      "coverage": 1.0,
      "random": 0.116,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "37": {
      "raw": 0.58,
      "skill": 0.4732,
      "coverage": 1.0,
      "random": 0.2027,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "38": {
      "raw": 0.3929,
      "skill": 0.3735,
      "coverage": 1.0,
      "random": 0.031,
      "rule": "chance",
      "in_index": true
    },
    "39": {
      "raw": 0.94,
      "skill": 0.9118,
      "coverage": 1.0,
      "random": 0.3201,
      "rule": "chance",
      "in_index": true
    },
    "40": {
      "raw": 0.6364,
      "skill": 0.5322,
      "coverage": 1.0,
      "random": 0.2227,
      "rule": "track",
      "in_index": true,
      "tracks": [
        {
          "track": "A · Arabic",
          "score": 0.4409,
          "headline": false
        },
        {
          "track": "A · English",
          "score": 0.6364,
          "headline": true
        },
        {
          "track": "C · Arabic pairs",
          "score": 0.81,
          "headline": false
        },
        {
          "track": "C · English pairs",
          "score": 0.96,
          "headline": false
        }
      ]
    },
    "41": {
      "raw": 0.793,
      "skill": 0.6894,
      "coverage": 1.0,
      "random": 0.3333,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "42": {
      "raw": 0.8411,
      "skill": 0.6909,
      "coverage": 1.0,
      "random": 0.486,
      "rule": "chance",
      "in_index": true
    },
    "43": {
      "raw": 0.7895,
      "skill": 0.666,
      "coverage": 1.0,
      "random": 0.3697,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "44": {
      "raw": 0.7788,
      "skill": 0.5576,
      "coverage": 1.0,
      "random": 0.5,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "45": {
      "raw": 0.1577,
      "skill": 0.0,
      "coverage": 1.0,
      "random": 0.1641,
      "rule": "chance",
      "in_index": true
    },
    "48": {
      "raw": 0.2936,
      "skill": 0.2936,
      "coverage": 1.0,
      "random": 0.25,
      "rule": "vs baseline",
      "in_index": true
    },
    "50": {
      "raw": 0.4421,
      "skill": 0.1903,
      "coverage": 1.0,
      "random": 0.311,
      "rule": "track",
      "in_index": true,
      "tracks": []
    },
    "56": {
      "raw": 0.692,
      "skill": 0.384,
      "coverage": 1.0,
      "random": 0.5,
      "rule": "chance",
      "in_index": true
    },
    "57": {
      "raw": 0.817,
      "skill": 0.7942,
      "coverage": 1.0,
      "random": 0.1109,
      "rule": "chance",
      "in_index": true
    },
    "58": {
      "raw": 0.7954,
      "skill": 0.7034,
      "coverage": 1.0,
      "random": 0.3101,
      "rule": "chance",
      "in_index": true
    },
    "59": {
      "raw": 0.8405,
      "skill": 0.6693,
      "coverage": 1.0,
      "random": 0.5177,
      "rule": "chance",
      "in_index": true
    },
    "61": {
      "raw": 0.8632,
      "skill": 0.7264,
      "coverage": 1.0,
      "random": 0.5,
      "rule": "chance",
      "in_index": true
    },
    "62": {
      "raw": 0.8612,
      "skill": 0.8149,
      "coverage": 1.0,
      "random": 0.25,
      "rule": "chance",
      "in_index": true
    },
    "64": {
      "raw": 0.7083,
      "skill": 0.6354,
      "coverage": 1.0,
      "random": 0.2,
      "rule": "chance",
      "in_index": true
    },
    "6": {
      "raw": 0.7963,
      "skill": 0.5224,
      "coverage": 1.0,
      "random": 0.5735,
      "rule": "shown, not counted",
      "in_index": false
    },
    "10": {
      "raw": 0.4419,
      "skill": 0.0714,
      "coverage": 1.0,
      "random": 0.399,
      "rule": "shown, not counted",
      "in_index": false
    },
    "24": {
      "raw": 0.8823,
      "skill": 0.8431,
      "coverage": 1.0,
      "random": 0.25,
      "rule": "shown, not counted",
      "in_index": false
    },
    "26": {
      "raw": 0.9895,
      "skill": 0.986,
      "coverage": 1.0,
      "random": 0.2502,
      "rule": "shown, not counted",
      "in_index": false
    },
    "27": {
      "raw": 0.9701,
      "skill": 0.9601,
      "coverage": 1.0,
      "random": 0.2502,
      "rule": "shown, not counted",
      "in_index": false
    },
    "34": {
      "raw": 0.3,
      "skill": 0.16,
      "coverage": 1.0,
      "random": 0.1667,
      "rule": "shown, not counted",
      "in_index": false
    }
  },
  "benchmarks": {
    "1": {
      "catalog_id": 1,
      "dataset": "BFCL",
      "requests": 1694,
      "answered": 1694,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 1694,
      "metric": "case exact accuracy",
      "score": 0.9669,
      "reference_same_cases": null,
      "median_ms": 139.6,
      "index_raw": 0.9669,
      "index_skill": 0.9554,
      "coverage": 1.0,
      "chance": 0.2592,
      "in_index": true
    },
    "2": {
      "catalog_id": 2,
      "dataset": "ToolRet",
      "requests": 685,
      "answered": 685,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 685,
      "metric": "nDCG@10",
      "score": 0.676,
      "reference_same_cases": null,
      "median_ms": 1421.8,
      "index_raw": 0.676,
      "index_skill": 0.6258,
      "coverage": 1.0,
      "chance": 0.1341,
      "in_index": true
    },
    "3": {
      "catalog_id": 3,
      "dataset": "API-Bank",
      "requests": 508,
      "answered": 508,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 508,
      "metric": "accuracy",
      "score": 0.8465,
      "reference_same_cases": null,
      "median_ms": 515.2,
      "index_raw": 0.8465,
      "index_skill": 0.8435,
      "coverage": 1.0,
      "chance": 0.0189,
      "in_index": true
    },
    "4": {
      "catalog_id": 4,
      "dataset": "BANKING77",
      "requests": 3080,
      "answered": 3080,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 3080,
      "metric": "macro-F1",
      "score": 0.8953,
      "reference_same_cases": null,
      "median_ms": 101.7,
      "index_raw": 0.8953,
      "index_skill": 0.894,
      "coverage": 1.0,
      "chance": 0.0127,
      "in_index": true
    },
    "5": {
      "catalog_id": 5,
      "dataset": "CLINC150+OOS",
      "requests": 5500,
      "answered": 5500,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 5500,
      "metric": "macro-F1",
      "score": 0.912,
      "reference_same_cases": null,
      "median_ms": 140.6,
      "index_raw": 0.912,
      "index_skill": 0.9115,
      "coverage": 1.0,
      "chance": 0.006,
      "in_index": true
    },
    "6": {
      "catalog_id": 6,
      "dataset": "RouterBench",
      "requests": 10000,
      "answered": 10000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 10000,
      "metric": "selected quality (quality objective)",
      "score": 0.7963,
      "reference_same_cases": null,
      "median_ms": 329.9,
      "tracks": {
        "RouterBench-0shot": {
          "metric": "selected quality (quality objective)",
          "score": 0.7838996602038777,
          "scored_requests": 5003
        },
        "RouterBench-5shot": {
          "metric": "selected quality (quality objective)",
          "score": 0.8086351811086652,
          "scored_requests": 4997
        }
      },
      "index_raw": 0.7963,
      "index_skill": 0.5224,
      "coverage": 1.0,
      "chance": 0.5735,
      "in_index": false
    },
    "9": {
      "catalog_id": 9,
      "dataset": "Home appliance simulator",
      "requests": 88,
      "answered": 88,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 88,
      "metric": "case exact accuracy",
      "score": 0.7614,
      "reference_same_cases": null,
      "median_ms": 1331.5,
      "index_raw": 0.7614,
      "index_skill": 0.7614,
      "coverage": 1.0,
      "chance": 0.0,
      "in_index": true
    },
    "10": {
      "catalog_id": 10,
      "dataset": "SGD/SGD-X",
      "requests": 2500,
      "answered": 2500,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 2500,
      "metric": "macro-F1",
      "score": 0.4419,
      "reference_same_cases": null,
      "median_ms": 119.9,
      "index_raw": 0.4419,
      "index_skill": 0.0714,
      "coverage": 1.0,
      "chance": 0.399,
      "in_index": false
    },
    "11": {
      "catalog_id": 11,
      "dataset": "ContractNLI",
      "requests": 123,
      "answered": 123,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 123,
      "metric": "macro-F1",
      "score": 0.8323,
      "reference_same_cases": null,
      "median_ms": 2119.6,
      "index_raw": 0.8323,
      "index_skill": 0.7574,
      "coverage": 1.0,
      "chance": 0.3085,
      "in_index": true
    },
    "12": {
      "catalog_id": 12,
      "dataset": "ANLI",
      "requests": 3200,
      "answered": 3200,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 3200,
      "metric": "macro-F1",
      "score": 0.7372,
      "reference_same_cases": null,
      "median_ms": 63.7,
      "index_raw": 0.7372,
      "index_skill": 0.6064,
      "coverage": 1.0,
      "chance": 0.3324,
      "in_index": true
    },
    "20": {
      "catalog_id": 20,
      "dataset": "BPoMP",
      "requests": 5000,
      "answered": 5000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 5000,
      "metric": "accuracy",
      "score": 0.9446,
      "reference_same_cases": null,
      "median_ms": 63.8,
      "index_raw": 0.9467,
      "index_skill": 0.8934,
      "coverage": 1.0,
      "chance": 0.5,
      "in_index": true
    },
    "21": {
      "catalog_id": 21,
      "dataset": "Humicroedit",
      "requests": 2628,
      "answered": 2628,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 2628,
      "metric": "accuracy",
      "score": 0.6191,
      "reference_same_cases": null,
      "median_ms": 62.9,
      "index_raw": 0.6191,
      "index_skill": 0.2382,
      "coverage": 1.0,
      "chance": 0.5,
      "in_index": true
    },
    "22": {
      "catalog_id": 22,
      "dataset": "POP909-CL",
      "requests": 2000,
      "answered": 2000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 2000,
      "metric": "accuracy",
      "score": 0.422,
      "reference_same_cases": null,
      "median_ms": 439.8,
      "index_raw": 0.4118,
      "index_skill": 0.4072,
      "coverage": 1.0,
      "chance": 0.0078,
      "in_index": true
    },
    "23": {
      "catalog_id": 23,
      "dataset": "cfcolor",
      "requests": 5000,
      "answered": 5000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 5000,
      "metric": "accuracy",
      "score": 0.654,
      "reference_same_cases": null,
      "median_ms": 76.4,
      "index_raw": 0.6553,
      "index_skill": 0.3106,
      "coverage": 1.0,
      "chance": 0.5,
      "in_index": true
    },
    "24": {
      "catalog_id": 24,
      "dataset": "MMLU",
      "requests": 14033,
      "answered": 14033,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 14033,
      "metric": "accuracy",
      "score": 0.8823,
      "reference_same_cases": null,
      "median_ms": 63.3,
      "index_raw": 0.8823,
      "index_skill": 0.8431,
      "coverage": 1.0,
      "chance": 0.25,
      "in_index": false
    },
    "25": {
      "catalog_id": 25,
      "dataset": "GPQA Diamond",
      "requests": 196,
      "answered": 196,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 196,
      "metric": "accuracy",
      "score": 0.4847,
      "reference_same_cases": null,
      "median_ms": 64.6,
      "index_raw": 0.4847,
      "index_skill": 0.3129,
      "coverage": 1.0,
      "chance": 0.25,
      "in_index": true
    },
    "26": {
      "catalog_id": 26,
      "dataset": "ARC-Easy",
      "requests": 2376,
      "answered": 2376,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 2376,
      "metric": "accuracy",
      "score": 0.9895,
      "reference_same_cases": null,
      "median_ms": 62.8,
      "index_raw": 0.9895,
      "index_skill": 0.986,
      "coverage": 1.0,
      "chance": 0.2502,
      "in_index": false
    },
    "27": {
      "catalog_id": 27,
      "dataset": "ARC-Challenge",
      "requests": 1172,
      "answered": 1172,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 1172,
      "metric": "accuracy",
      "score": 0.9701,
      "reference_same_cases": null,
      "median_ms": 63.0,
      "index_raw": 0.9701,
      "index_skill": 0.9601,
      "coverage": 1.0,
      "chance": 0.2502,
      "in_index": false
    },
    "28": {
      "catalog_id": 28,
      "dataset": "WinoGrande",
      "requests": 1267,
      "answered": 1267,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 1267,
      "metric": "accuracy",
      "score": 0.8603,
      "reference_same_cases": null,
      "median_ms": 62.6,
      "index_raw": 0.8603,
      "index_skill": 0.7206,
      "coverage": 1.0,
      "chance": 0.5,
      "in_index": true
    },
    "29": {
      "catalog_id": 29,
      "dataset": "HellaSwag",
      "requests": 10042,
      "answered": 10042,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 10042,
      "metric": "accuracy",
      "score": 0.952,
      "reference_same_cases": null,
      "median_ms": 67.6,
      "index_raw": 0.952,
      "index_skill": 0.936,
      "coverage": 1.0,
      "chance": 0.25,
      "in_index": true
    },
    "30": {
      "catalog_id": 30,
      "dataset": "GSM8K",
      "requests": 2638,
      "answered": 2638,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 2638,
      "metric": "accuracy",
      "score": 0.7953,
      "reference_same_cases": null,
      "median_ms": 63.8,
      "tracks": [
        {
          "track": "GSM8K-4choice",
          "score": 0.8514,
          "headline": false
        },
        {
          "track": "GSM8K-10choice",
          "score": 0.7392,
          "headline": false
        }
      ],
      "index_raw": 0.7953,
      "index_skill": 0.756,
      "coverage": 1.0,
      "chance": 0.25,
      "in_index": true
    },
    "31": {
      "catalog_id": 31,
      "dataset": "ChessBench",
      "requests": 5000,
      "answered": 5000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 5000,
      "metric": "accuracy",
      "score": 0.207,
      "reference_same_cases": null,
      "median_ms": 104.9,
      "index_raw": 0.207,
      "index_skill": 0.1363,
      "coverage": 1.0,
      "chance": 0.0819,
      "in_index": true
    },
    "32": {
      "catalog_id": 32,
      "dataset": "MuSR",
      "requests": 752,
      "answered": 752,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 752,
      "metric": "accuracy",
      "score": 0.6742,
      "reference_same_cases": null,
      "median_ms": 109.7,
      "index_raw": 0.6742,
      "index_skill": 0.482,
      "coverage": 1.0,
      "chance": 0.371,
      "in_index": true
    },
    "33": {
      "catalog_id": 33,
      "dataset": "SATA-Bench",
      "requests": 1650,
      "answered": 1650,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 1650,
      "metric": "case exact accuracy",
      "score": 0.2558,
      "reference_same_cases": null,
      "median_ms": 264.1,
      "index_raw": 0.2558,
      "index_skill": 0.2459,
      "coverage": 1.0,
      "chance": 0.0131,
      "in_index": true
    },
    "34": {
      "catalog_id": 34,
      "dataset": "SimpleBench",
      "requests": 10,
      "answered": 10,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 10,
      "metric": "accuracy",
      "score": 0.3,
      "reference_same_cases": null,
      "median_ms": 63.8,
      "index_raw": 0.3,
      "index_skill": 0.16,
      "coverage": 1.0,
      "chance": 0.1667,
      "in_index": false
    },
    "36": {
      "catalog_id": 36,
      "dataset": "BRIGHT",
      "requests": 220,
      "answered": 220,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 220,
      "metric": "nDCG@10",
      "score": 0.4851,
      "reference_same_cases": null,
      "median_ms": 2529.1,
      "index_raw": 0.4851,
      "index_skill": 0.4175,
      "coverage": 1.0,
      "chance": 0.116,
      "in_index": true
    },
    "37": {
      "catalog_id": 37,
      "dataset": "Amazon ESCI",
      "requests": 5000,
      "answered": 5000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 5000,
      "metric": "macro-F1",
      "score": 0.58,
      "reference_same_cases": null,
      "median_ms": 72.5,
      "index_raw": 0.58,
      "index_skill": 0.4732,
      "coverage": 1.0,
      "chance": 0.2027,
      "in_index": true
    },
    "38": {
      "catalog_id": 38,
      "dataset": "ACOS",
      "requests": 1565,
      "answered": 1565,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 1565,
      "metric": "per-review F1",
      "score": 0.3929,
      "reference_same_cases": null,
      "median_ms": 778.1,
      "index_raw": 0.3929,
      "index_skill": 0.3735,
      "coverage": 1.0,
      "chance": 0.031,
      "in_index": true
    },
    "39": {
      "catalog_id": 39,
      "dataset": "FinEntity",
      "requests": 979,
      "answered": 979,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 979,
      "metric": "macro-F1",
      "score": 0.94,
      "reference_same_cases": null,
      "median_ms": 74.5,
      "index_raw": 0.94,
      "index_skill": 0.9118,
      "coverage": 1.0,
      "chance": 0.3201,
      "in_index": true
    },
    "40": {
      "catalog_id": 40,
      "dataset": "iSarcasmEval",
      "requests": 4600,
      "answered": 4600,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 4600,
      "metric": "Sarcasm F1 · track A, English",
      "score": 0.6364,
      "reference_same_cases": null,
      "median_ms": 63.6,
      "tracks": [
        {
          "track": "A · Arabic",
          "score": 0.4409,
          "headline": false
        },
        {
          "track": "A · English",
          "score": 0.6364,
          "headline": true
        },
        {
          "track": "C · Arabic pairs",
          "score": 0.81,
          "headline": false
        },
        {
          "track": "C · English pairs",
          "score": 0.96,
          "headline": false
        }
      ],
      "index_raw": 0.6364,
      "index_skill": 0.5322,
      "coverage": 1.0,
      "chance": 0.2227,
      "in_index": true
    },
    "41": {
      "catalog_id": 41,
      "dataset": "VAST",
      "requests": 3006,
      "answered": 3006,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 3006,
      "metric": "macro-F1",
      "score": 0.793,
      "reference_same_cases": null,
      "median_ms": 64.5,
      "index_raw": 0.793,
      "index_skill": 0.6894,
      "coverage": 1.0,
      "chance": 0.3333,
      "in_index": true
    },
    "42": {
      "catalog_id": 42,
      "dataset": "NLI4CT",
      "requests": 5500,
      "answered": 5500,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 5500,
      "metric": "macro-F1",
      "score": 0.8411,
      "reference_same_cases": null,
      "median_ms": 78.5,
      "index_raw": 0.8411,
      "index_skill": 0.6909,
      "coverage": 1.0,
      "chance": 0.486,
      "in_index": true
    },
    "43": {
      "catalog_id": 43,
      "dataset": "CRUXEval",
      "requests": 570,
      "answered": 570,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 570,
      "metric": "accuracy",
      "score": 0.7895,
      "reference_same_cases": null,
      "median_ms": 63.4,
      "index_raw": 0.7895,
      "index_skill": 0.666,
      "coverage": 1.0,
      "chance": 0.3697,
      "in_index": true
    },
    "44": {
      "catalog_id": 44,
      "dataset": "CLadder",
      "requests": 5000,
      "answered": 5000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 5000,
      "metric": "accuracy",
      "score": 0.7788,
      "reference_same_cases": null,
      "median_ms": 64.0,
      "index_raw": 0.7788,
      "index_skill": 0.5576,
      "coverage": 1.0,
      "chance": 0.5,
      "in_index": true
    },
    "45": {
      "catalog_id": 45,
      "dataset": "HLE",
      "requests": 501,
      "answered": 501,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 501,
      "metric": "accuracy",
      "score": 0.1577,
      "reference_same_cases": null,
      "median_ms": 67.6,
      "index_raw": 0.1577,
      "index_skill": 0.0,
      "coverage": 1.0,
      "chance": 0.1641,
      "in_index": true
    },
    "48": {
      "catalog_id": 48,
      "dataset": "ForecastBench",
      "requests": 10139,
      "answered": 10139,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 10139,
      "metric": "Brier (lower is better)",
      "score": 0.1766,
      "reference_same_cases": null,
      "median_ms": 92.7,
      "index_raw": 0.2936,
      "index_skill": 0.2936,
      "coverage": 1.0,
      "chance": 0.25,
      "in_index": true
    },
    "50": {
      "catalog_id": 50,
      "dataset": "Habermas Machine",
      "requests": 1676,
      "answered": 1676,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "scored_requests": 1676,
      "metric": "accuracy",
      "score": 0.4421,
      "reference_same_cases": null,
      "median_ms": 102.4,
      "index_raw": 0.4421,
      "index_skill": 0.1903,
      "coverage": 1.0,
      "chance": 0.311,
      "in_index": true
    },
    "56": {
      "catalog_id": 56,
      "dataset": "PhishNChips phishing decisions",
      "requests": 2000,
      "answered": 2000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "metric": "accuracy",
      "score": 0.692,
      "median_ms": 196.0,
      "scored_requests": 2000,
      "index_raw": 0.692,
      "index_skill": 0.384,
      "coverage": 1.0,
      "chance": 0.5,
      "in_index": true
    },
    "57": {
      "catalog_id": 57,
      "dataset": "MMLU-Pro",
      "requests": 12032,
      "answered": 12032,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "metric": "accuracy",
      "score": 0.817,
      "median_ms": 64.5,
      "scored_requests": 12032,
      "index_raw": 0.817,
      "index_skill": 0.7942,
      "coverage": 1.0,
      "chance": 0.1109,
      "in_index": true
    },
    "58": {
      "catalog_id": 58,
      "dataset": "BBH fixed-option tasks",
      "requests": 5507,
      "answered": 5507,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "metric": "accuracy",
      "score": 0.7954,
      "median_ms": 64.0,
      "scored_requests": 5507,
      "index_raw": 0.7954,
      "index_skill": 0.7034,
      "coverage": 1.0,
      "chance": 0.3101,
      "in_index": true
    },
    "59": {
      "catalog_id": 59,
      "dataset": "RAGTruth response-level hallucination",
      "requests": 2700,
      "answered": 2700,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "metric": "F1 on hallucinated class",
      "score": 0.8405,
      "median_ms": 95.3,
      "scored_requests": 2700,
      "index_raw": 0.8405,
      "index_skill": 0.6693,
      "coverage": 1.0,
      "chance": 0.5177,
      "in_index": true
    },
    "61": {
      "catalog_id": 61,
      "dataset": "HoVer claim verification",
      "requests": 4000,
      "answered": 4000,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "metric": "accuracy",
      "score": 0.8632,
      "median_ms": 75.6,
      "scored_requests": 4000,
      "index_raw": 0.8632,
      "index_skill": 0.7264,
      "coverage": 1.0,
      "chance": 0.5,
      "in_index": true
    },
    "62": {
      "catalog_id": 62,
      "dataset": "When2Call MCQ",
      "requests": 3652,
      "answered": 3652,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "metric": "accuracy",
      "score": 0.8612,
      "median_ms": 93.7,
      "scored_requests": 3652,
      "index_raw": 0.8612,
      "index_skill": 0.8149,
      "coverage": 1.0,
      "chance": 0.25,
      "in_index": true
    },
    "64": {
      "catalog_id": 64,
      "dataset": "New Yorker caption matching",
      "requests": 528,
      "answered": 528,
      "unsupported": 0,
      "errors": 0,
      "abstained": 0,
      "pending": 0,
      "metric": "accuracy",
      "score": 0.7083,
      "median_ms": 63.8,
      "scored_requests": 528,
      "index_raw": 0.7083,
      "index_skill": 0.6354,
      "coverage": 1.0,
      "chance": 0.2,
      "in_index": true
    }
  },
  "panel_id": "decision-index-0.2.1",
  "note": "Decision Index 0.2.1 averages 38 benchmarks in five areas. Arts & Human Taste weighs 10%; the other four share 90% in proportion to the square root of their benchmark count (knowledge 25.8%, language 25.8%, retrieval 20.0%, tools 18.3%). Inside an area, gold ★ benchmarks weigh 1.2 and the rest 1.0; the index is 100 x the weighted mean of the five areas. Each benchmark is chance-corrected first, (score - chance) / (1 - chance) clipped to 0-1, so 0 means random guessing and 100 means perfect. Every score is coverage-adjusted, so an unanswered or unsupported request counts as wrong. ForecastBench enters against its baseline: clip((0.25 - Brier) / 0.25) x coverage, so always predicting 0.5 scores zero. MMLU, ARC-Easy, ARC-Challenge, RouterBench, SGD stay on the board as non-index benchmarks. The six interactive environments are still unrun and stay out. Every entrant on the board has results on all 38 index benchmarks. Point estimates only, no uncertainty intervals yet."
}