| { |
| "engine": "matilda-jev-di62p43-fp4", |
| "edition": "0.2.1", |
| "generated_utc": "2026-10-06T16:58:26+00:00", |
| "suite": { |
| "edition": "release-v2.1", |
| "requests": 120340, |
| "scoreable": 119898, |
| "excluded": 442, |
| "added_requests": 30419, |
| "benchmarks": 44, |
| "rows_sha256": "b2b56d6fb636837ca469e689087bdbf373dda8de7638aa2da6793e6eda0792d5", |
| "added_sha256": "7429f3c9cdddb772c1cfc42bb2a45e8516b0032152b746e6929f1c8b52f4ce89" |
| }, |
| "completed": 150317, |
| "complete": true, |
| "counts": { |
| "ok": 150317 |
| }, |
| "latency_ms": { |
| "median": 71.3, |
| "p95": 408.4, |
| "mean": 137.5 |
| }, |
| "decision_index": 61.77, |
| "raw_index": 70.9, |
| "scores": { |
| "balanced_skill": 61.77, |
| "balanced_raw": 70.9, |
| "breadth_skill": 60.48 |
| }, |
| "areas": [ |
| { |
| "id": "knowledge", |
| "label": "Knowledge & Reasoning", |
| "raw": 0.5747, |
| "skill": 0.4645, |
| "coverage": 1.0, |
| "n": 10, |
| "benchmarks": [ |
| 25, |
| 30, |
| 31, |
| 32, |
| 33, |
| 43, |
| 44, |
| 45, |
| 57, |
| 58 |
| ] |
| }, |
| { |
| "id": "language", |
| "label": "Language Understanding", |
| "raw": 0.7864, |
| "skill": 0.6924, |
| "coverage": 1.0, |
| "n": 10, |
| "benchmarks": [ |
| 11, |
| 12, |
| 28, |
| 29, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 59 |
| ] |
| }, |
| { |
| "id": "retrieval", |
| "label": "Retrieval & Classification", |
| "raw": 0.7403, |
| "skill": 0.6441, |
| "coverage": 1.0, |
| "n": 6, |
| "benchmarks": [ |
| 4, |
| 5, |
| 36, |
| 37, |
| 56, |
| 61 |
| ] |
| }, |
| { |
| "id": "tools", |
| "label": "Tools & Automation", |
| "raw": 0.8286, |
| "skill": 0.8076, |
| "coverage": 1.0, |
| "n": 5, |
| "benchmarks": [ |
| 1, |
| 2, |
| 3, |
| 9, |
| 62 |
| ] |
| }, |
| { |
| "id": "arts", |
| "label": "Arts & Human Taste", |
| "raw": 0.5744, |
| "skill": 0.4205, |
| "coverage": 1.0, |
| "n": 7, |
| "benchmarks": [ |
| 20, |
| 21, |
| 22, |
| 23, |
| 48, |
| 50, |
| 64 |
| ] |
| } |
| ], |
| "index_benchmarks": { |
| "1": { |
| "raw": 0.9669, |
| "skill": 0.9554, |
| "coverage": 1.0, |
| "random": 0.2592, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "2": { |
| "raw": 0.676, |
| "skill": 0.6258, |
| "coverage": 1.0, |
| "random": 0.1341, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "3": { |
| "raw": 0.8465, |
| "skill": 0.8435, |
| "coverage": 1.0, |
| "random": 0.0189, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "4": { |
| "raw": 0.8953, |
| "skill": 0.894, |
| "coverage": 1.0, |
| "random": 0.0127, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "5": { |
| "raw": 0.912, |
| "skill": 0.9115, |
| "coverage": 1.0, |
| "random": 0.006, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "9": { |
| "raw": 0.7614, |
| "skill": 0.7614, |
| "coverage": 1.0, |
| "random": 0.0, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "11": { |
| "raw": 0.8323, |
| "skill": 0.7574, |
| "coverage": 1.0, |
| "random": 0.3085, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "12": { |
| "raw": 0.7372, |
| "skill": 0.6064, |
| "coverage": 1.0, |
| "random": 0.3324, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "20": { |
| "raw": 0.9467, |
| "skill": 0.8934, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "21": { |
| "raw": 0.6191, |
| "skill": 0.2382, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "22": { |
| "raw": 0.4118, |
| "skill": 0.4072, |
| "coverage": 1.0, |
| "random": 0.0078, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "23": { |
| "raw": 0.6553, |
| "skill": 0.3106, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "25": { |
| "raw": 0.4847, |
| "skill": 0.3129, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "28": { |
| "raw": 0.8603, |
| "skill": 0.7206, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "29": { |
| "raw": 0.952, |
| "skill": 0.936, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "30": { |
| "raw": 0.7953, |
| "skill": 0.756, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "score": 0.8514, |
| "headline": false |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "score": 0.7392, |
| "headline": false |
| } |
| ] |
| }, |
| "31": { |
| "raw": 0.207, |
| "skill": 0.1363, |
| "coverage": 1.0, |
| "random": 0.0819, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "32": { |
| "raw": 0.6742, |
| "skill": 0.482, |
| "coverage": 1.0, |
| "random": 0.371, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "33": { |
| "raw": 0.2558, |
| "skill": 0.2459, |
| "coverage": 1.0, |
| "random": 0.0131, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "36": { |
| "raw": 0.4851, |
| "skill": 0.4175, |
| "coverage": 1.0, |
| "random": 0.116, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "37": { |
| "raw": 0.58, |
| "skill": 0.4732, |
| "coverage": 1.0, |
| "random": 0.2027, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "38": { |
| "raw": 0.3929, |
| "skill": 0.3735, |
| "coverage": 1.0, |
| "random": 0.031, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "39": { |
| "raw": 0.94, |
| "skill": 0.9118, |
| "coverage": 1.0, |
| "random": 0.3201, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "40": { |
| "raw": 0.6364, |
| "skill": 0.5322, |
| "coverage": 1.0, |
| "random": 0.2227, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [ |
| { |
| "track": "A · Arabic", |
| "score": 0.4409, |
| "headline": false |
| }, |
| { |
| "track": "A · English", |
| "score": 0.6364, |
| "headline": true |
| }, |
| { |
| "track": "C · Arabic pairs", |
| "score": 0.81, |
| "headline": false |
| }, |
| { |
| "track": "C · English pairs", |
| "score": 0.96, |
| "headline": false |
| } |
| ] |
| }, |
| "41": { |
| "raw": 0.793, |
| "skill": 0.6894, |
| "coverage": 1.0, |
| "random": 0.3333, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "42": { |
| "raw": 0.8411, |
| "skill": 0.6909, |
| "coverage": 1.0, |
| "random": 0.486, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "43": { |
| "raw": 0.7895, |
| "skill": 0.666, |
| "coverage": 1.0, |
| "random": 0.3697, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "44": { |
| "raw": 0.7788, |
| "skill": 0.5576, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "45": { |
| "raw": 0.1577, |
| "skill": 0.0, |
| "coverage": 1.0, |
| "random": 0.1641, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "48": { |
| "raw": 0.2936, |
| "skill": 0.2936, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "vs baseline", |
| "in_index": true |
| }, |
| "50": { |
| "raw": 0.4421, |
| "skill": 0.1903, |
| "coverage": 1.0, |
| "random": 0.311, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "56": { |
| "raw": 0.692, |
| "skill": 0.384, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "57": { |
| "raw": 0.817, |
| "skill": 0.7942, |
| "coverage": 1.0, |
| "random": 0.1109, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "58": { |
| "raw": 0.7954, |
| "skill": 0.7034, |
| "coverage": 1.0, |
| "random": 0.3101, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "59": { |
| "raw": 0.8405, |
| "skill": 0.6693, |
| "coverage": 1.0, |
| "random": 0.5177, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "61": { |
| "raw": 0.8632, |
| "skill": 0.7264, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "62": { |
| "raw": 0.8612, |
| "skill": 0.8149, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "64": { |
| "raw": 0.7083, |
| "skill": 0.6354, |
| "coverage": 1.0, |
| "random": 0.2, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "6": { |
| "raw": 0.7963, |
| "skill": 0.5224, |
| "coverage": 1.0, |
| "random": 0.5735, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "10": { |
| "raw": 0.4419, |
| "skill": 0.0714, |
| "coverage": 1.0, |
| "random": 0.399, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "24": { |
| "raw": 0.8823, |
| "skill": 0.8431, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "26": { |
| "raw": 0.9895, |
| "skill": 0.986, |
| "coverage": 1.0, |
| "random": 0.2502, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "27": { |
| "raw": 0.9701, |
| "skill": 0.9601, |
| "coverage": 1.0, |
| "random": 0.2502, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "34": { |
| "raw": 0.3, |
| "skill": 0.16, |
| "coverage": 1.0, |
| "random": 0.1667, |
| "rule": "shown, not counted", |
| "in_index": false |
| } |
| }, |
| "benchmarks": { |
| "1": { |
| "catalog_id": 1, |
| "dataset": "BFCL", |
| "requests": 1694, |
| "answered": 1694, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1694, |
| "metric": "case exact accuracy", |
| "score": 0.9669, |
| "reference_same_cases": null, |
| "median_ms": 139.6, |
| "index_raw": 0.9669, |
| "index_skill": 0.9554, |
| "coverage": 1.0, |
| "chance": 0.2592, |
| "in_index": true |
| }, |
| "2": { |
| "catalog_id": 2, |
| "dataset": "ToolRet", |
| "requests": 685, |
| "answered": 685, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 685, |
| "metric": "nDCG@10", |
| "score": 0.676, |
| "reference_same_cases": null, |
| "median_ms": 1421.8, |
| "index_raw": 0.676, |
| "index_skill": 0.6258, |
| "coverage": 1.0, |
| "chance": 0.1341, |
| "in_index": true |
| }, |
| "3": { |
| "catalog_id": 3, |
| "dataset": "API-Bank", |
| "requests": 508, |
| "answered": 508, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 508, |
| "metric": "accuracy", |
| "score": 0.8465, |
| "reference_same_cases": null, |
| "median_ms": 515.2, |
| "index_raw": 0.8465, |
| "index_skill": 0.8435, |
| "coverage": 1.0, |
| "chance": 0.0189, |
| "in_index": true |
| }, |
| "4": { |
| "catalog_id": 4, |
| "dataset": "BANKING77", |
| "requests": 3080, |
| "answered": 3080, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 3080, |
| "metric": "macro-F1", |
| "score": 0.8953, |
| "reference_same_cases": null, |
| "median_ms": 101.7, |
| "index_raw": 0.8953, |
| "index_skill": 0.894, |
| "coverage": 1.0, |
| "chance": 0.0127, |
| "in_index": true |
| }, |
| "5": { |
| "catalog_id": 5, |
| "dataset": "CLINC150+OOS", |
| "requests": 5500, |
| "answered": 5500, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5500, |
| "metric": "macro-F1", |
| "score": 0.912, |
| "reference_same_cases": null, |
| "median_ms": 140.6, |
| "index_raw": 0.912, |
| "index_skill": 0.9115, |
| "coverage": 1.0, |
| "chance": 0.006, |
| "in_index": true |
| }, |
| "6": { |
| "catalog_id": 6, |
| "dataset": "RouterBench", |
| "requests": 10000, |
| "answered": 10000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10000, |
| "metric": "selected quality (quality objective)", |
| "score": 0.7963, |
| "reference_same_cases": null, |
| "median_ms": 329.9, |
| "tracks": { |
| "RouterBench-0shot": { |
| "metric": "selected quality (quality objective)", |
| "score": 0.7838996602038777, |
| "scored_requests": 5003 |
| }, |
| "RouterBench-5shot": { |
| "metric": "selected quality (quality objective)", |
| "score": 0.8086351811086652, |
| "scored_requests": 4997 |
| } |
| }, |
| "index_raw": 0.7963, |
| "index_skill": 0.5224, |
| "coverage": 1.0, |
| "chance": 0.5735, |
| "in_index": false |
| }, |
| "9": { |
| "catalog_id": 9, |
| "dataset": "Home appliance simulator", |
| "requests": 88, |
| "answered": 88, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 88, |
| "metric": "case exact accuracy", |
| "score": 0.7614, |
| "reference_same_cases": null, |
| "median_ms": 1331.5, |
| "index_raw": 0.7614, |
| "index_skill": 0.7614, |
| "coverage": 1.0, |
| "chance": 0.0, |
| "in_index": true |
| }, |
| "10": { |
| "catalog_id": 10, |
| "dataset": "SGD/SGD-X", |
| "requests": 2500, |
| "answered": 2500, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2500, |
| "metric": "macro-F1", |
| "score": 0.4419, |
| "reference_same_cases": null, |
| "median_ms": 119.9, |
| "index_raw": 0.4419, |
| "index_skill": 0.0714, |
| "coverage": 1.0, |
| "chance": 0.399, |
| "in_index": false |
| }, |
| "11": { |
| "catalog_id": 11, |
| "dataset": "ContractNLI", |
| "requests": 123, |
| "answered": 123, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 123, |
| "metric": "macro-F1", |
| "score": 0.8323, |
| "reference_same_cases": null, |
| "median_ms": 2119.6, |
| "index_raw": 0.8323, |
| "index_skill": 0.7574, |
| "coverage": 1.0, |
| "chance": 0.3085, |
| "in_index": true |
| }, |
| "12": { |
| "catalog_id": 12, |
| "dataset": "ANLI", |
| "requests": 3200, |
| "answered": 3200, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 3200, |
| "metric": "macro-F1", |
| "score": 0.7372, |
| "reference_same_cases": null, |
| "median_ms": 63.7, |
| "index_raw": 0.7372, |
| "index_skill": 0.6064, |
| "coverage": 1.0, |
| "chance": 0.3324, |
| "in_index": true |
| }, |
| "20": { |
| "catalog_id": 20, |
| "dataset": "BPoMP", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.9446, |
| "reference_same_cases": null, |
| "median_ms": 63.8, |
| "index_raw": 0.9467, |
| "index_skill": 0.8934, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "21": { |
| "catalog_id": 21, |
| "dataset": "Humicroedit", |
| "requests": 2628, |
| "answered": 2628, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2628, |
| "metric": "accuracy", |
| "score": 0.6191, |
| "reference_same_cases": null, |
| "median_ms": 62.9, |
| "index_raw": 0.6191, |
| "index_skill": 0.2382, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "22": { |
| "catalog_id": 22, |
| "dataset": "POP909-CL", |
| "requests": 2000, |
| "answered": 2000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2000, |
| "metric": "accuracy", |
| "score": 0.422, |
| "reference_same_cases": null, |
| "median_ms": 439.8, |
| "index_raw": 0.4118, |
| "index_skill": 0.4072, |
| "coverage": 1.0, |
| "chance": 0.0078, |
| "in_index": true |
| }, |
| "23": { |
| "catalog_id": 23, |
| "dataset": "cfcolor", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.654, |
| "reference_same_cases": null, |
| "median_ms": 76.4, |
| "index_raw": 0.6553, |
| "index_skill": 0.3106, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "24": { |
| "catalog_id": 24, |
| "dataset": "MMLU", |
| "requests": 14033, |
| "answered": 14033, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 14033, |
| "metric": "accuracy", |
| "score": 0.8823, |
| "reference_same_cases": null, |
| "median_ms": 63.3, |
| "index_raw": 0.8823, |
| "index_skill": 0.8431, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": false |
| }, |
| "25": { |
| "catalog_id": 25, |
| "dataset": "GPQA Diamond", |
| "requests": 196, |
| "answered": 196, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 196, |
| "metric": "accuracy", |
| "score": 0.4847, |
| "reference_same_cases": null, |
| "median_ms": 64.6, |
| "index_raw": 0.4847, |
| "index_skill": 0.3129, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "26": { |
| "catalog_id": 26, |
| "dataset": "ARC-Easy", |
| "requests": 2376, |
| "answered": 2376, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2376, |
| "metric": "accuracy", |
| "score": 0.9895, |
| "reference_same_cases": null, |
| "median_ms": 62.8, |
| "index_raw": 0.9895, |
| "index_skill": 0.986, |
| "coverage": 1.0, |
| "chance": 0.2502, |
| "in_index": false |
| }, |
| "27": { |
| "catalog_id": 27, |
| "dataset": "ARC-Challenge", |
| "requests": 1172, |
| "answered": 1172, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1172, |
| "metric": "accuracy", |
| "score": 0.9701, |
| "reference_same_cases": null, |
| "median_ms": 63.0, |
| "index_raw": 0.9701, |
| "index_skill": 0.9601, |
| "coverage": 1.0, |
| "chance": 0.2502, |
| "in_index": false |
| }, |
| "28": { |
| "catalog_id": 28, |
| "dataset": "WinoGrande", |
| "requests": 1267, |
| "answered": 1267, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1267, |
| "metric": "accuracy", |
| "score": 0.8603, |
| "reference_same_cases": null, |
| "median_ms": 62.6, |
| "index_raw": 0.8603, |
| "index_skill": 0.7206, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "29": { |
| "catalog_id": 29, |
| "dataset": "HellaSwag", |
| "requests": 10042, |
| "answered": 10042, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10042, |
| "metric": "accuracy", |
| "score": 0.952, |
| "reference_same_cases": null, |
| "median_ms": 67.6, |
| "index_raw": 0.952, |
| "index_skill": 0.936, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "30": { |
| "catalog_id": 30, |
| "dataset": "GSM8K", |
| "requests": 2638, |
| "answered": 2638, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2638, |
| "metric": "accuracy", |
| "score": 0.7953, |
| "reference_same_cases": null, |
| "median_ms": 63.8, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "score": 0.8514, |
| "headline": false |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "score": 0.7392, |
| "headline": false |
| } |
| ], |
| "index_raw": 0.7953, |
| "index_skill": 0.756, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "31": { |
| "catalog_id": 31, |
| "dataset": "ChessBench", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.207, |
| "reference_same_cases": null, |
| "median_ms": 104.9, |
| "index_raw": 0.207, |
| "index_skill": 0.1363, |
| "coverage": 1.0, |
| "chance": 0.0819, |
| "in_index": true |
| }, |
| "32": { |
| "catalog_id": 32, |
| "dataset": "MuSR", |
| "requests": 752, |
| "answered": 752, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 752, |
| "metric": "accuracy", |
| "score": 0.6742, |
| "reference_same_cases": null, |
| "median_ms": 109.7, |
| "index_raw": 0.6742, |
| "index_skill": 0.482, |
| "coverage": 1.0, |
| "chance": 0.371, |
| "in_index": true |
| }, |
| "33": { |
| "catalog_id": 33, |
| "dataset": "SATA-Bench", |
| "requests": 1650, |
| "answered": 1650, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1650, |
| "metric": "case exact accuracy", |
| "score": 0.2558, |
| "reference_same_cases": null, |
| "median_ms": 264.1, |
| "index_raw": 0.2558, |
| "index_skill": 0.2459, |
| "coverage": 1.0, |
| "chance": 0.0131, |
| "in_index": true |
| }, |
| "34": { |
| "catalog_id": 34, |
| "dataset": "SimpleBench", |
| "requests": 10, |
| "answered": 10, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10, |
| "metric": "accuracy", |
| "score": 0.3, |
| "reference_same_cases": null, |
| "median_ms": 63.8, |
| "index_raw": 0.3, |
| "index_skill": 0.16, |
| "coverage": 1.0, |
| "chance": 0.1667, |
| "in_index": false |
| }, |
| "36": { |
| "catalog_id": 36, |
| "dataset": "BRIGHT", |
| "requests": 220, |
| "answered": 220, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 220, |
| "metric": "nDCG@10", |
| "score": 0.4851, |
| "reference_same_cases": null, |
| "median_ms": 2529.1, |
| "index_raw": 0.4851, |
| "index_skill": 0.4175, |
| "coverage": 1.0, |
| "chance": 0.116, |
| "in_index": true |
| }, |
| "37": { |
| "catalog_id": 37, |
| "dataset": "Amazon ESCI", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "macro-F1", |
| "score": 0.58, |
| "reference_same_cases": null, |
| "median_ms": 72.5, |
| "index_raw": 0.58, |
| "index_skill": 0.4732, |
| "coverage": 1.0, |
| "chance": 0.2027, |
| "in_index": true |
| }, |
| "38": { |
| "catalog_id": 38, |
| "dataset": "ACOS", |
| "requests": 1565, |
| "answered": 1565, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1565, |
| "metric": "per-review F1", |
| "score": 0.3929, |
| "reference_same_cases": null, |
| "median_ms": 778.1, |
| "index_raw": 0.3929, |
| "index_skill": 0.3735, |
| "coverage": 1.0, |
| "chance": 0.031, |
| "in_index": true |
| }, |
| "39": { |
| "catalog_id": 39, |
| "dataset": "FinEntity", |
| "requests": 979, |
| "answered": 979, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 979, |
| "metric": "macro-F1", |
| "score": 0.94, |
| "reference_same_cases": null, |
| "median_ms": 74.5, |
| "index_raw": 0.94, |
| "index_skill": 0.9118, |
| "coverage": 1.0, |
| "chance": 0.3201, |
| "in_index": true |
| }, |
| "40": { |
| "catalog_id": 40, |
| "dataset": "iSarcasmEval", |
| "requests": 4600, |
| "answered": 4600, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 4600, |
| "metric": "Sarcasm F1 · track A, English", |
| "score": 0.6364, |
| "reference_same_cases": null, |
| "median_ms": 63.6, |
| "tracks": [ |
| { |
| "track": "A · Arabic", |
| "score": 0.4409, |
| "headline": false |
| }, |
| { |
| "track": "A · English", |
| "score": 0.6364, |
| "headline": true |
| }, |
| { |
| "track": "C · Arabic pairs", |
| "score": 0.81, |
| "headline": false |
| }, |
| { |
| "track": "C · English pairs", |
| "score": 0.96, |
| "headline": false |
| } |
| ], |
| "index_raw": 0.6364, |
| "index_skill": 0.5322, |
| "coverage": 1.0, |
| "chance": 0.2227, |
| "in_index": true |
| }, |
| "41": { |
| "catalog_id": 41, |
| "dataset": "VAST", |
| "requests": 3006, |
| "answered": 3006, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 3006, |
| "metric": "macro-F1", |
| "score": 0.793, |
| "reference_same_cases": null, |
| "median_ms": 64.5, |
| "index_raw": 0.793, |
| "index_skill": 0.6894, |
| "coverage": 1.0, |
| "chance": 0.3333, |
| "in_index": true |
| }, |
| "42": { |
| "catalog_id": 42, |
| "dataset": "NLI4CT", |
| "requests": 5500, |
| "answered": 5500, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5500, |
| "metric": "macro-F1", |
| "score": 0.8411, |
| "reference_same_cases": null, |
| "median_ms": 78.5, |
| "index_raw": 0.8411, |
| "index_skill": 0.6909, |
| "coverage": 1.0, |
| "chance": 0.486, |
| "in_index": true |
| }, |
| "43": { |
| "catalog_id": 43, |
| "dataset": "CRUXEval", |
| "requests": 570, |
| "answered": 570, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 570, |
| "metric": "accuracy", |
| "score": 0.7895, |
| "reference_same_cases": null, |
| "median_ms": 63.4, |
| "index_raw": 0.7895, |
| "index_skill": 0.666, |
| "coverage": 1.0, |
| "chance": 0.3697, |
| "in_index": true |
| }, |
| "44": { |
| "catalog_id": 44, |
| "dataset": "CLadder", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.7788, |
| "reference_same_cases": null, |
| "median_ms": 64.0, |
| "index_raw": 0.7788, |
| "index_skill": 0.5576, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "45": { |
| "catalog_id": 45, |
| "dataset": "HLE", |
| "requests": 501, |
| "answered": 501, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 501, |
| "metric": "accuracy", |
| "score": 0.1577, |
| "reference_same_cases": null, |
| "median_ms": 67.6, |
| "index_raw": 0.1577, |
| "index_skill": 0.0, |
| "coverage": 1.0, |
| "chance": 0.1641, |
| "in_index": true |
| }, |
| "48": { |
| "catalog_id": 48, |
| "dataset": "ForecastBench", |
| "requests": 10139, |
| "answered": 10139, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10139, |
| "metric": "Brier (lower is better)", |
| "score": 0.1766, |
| "reference_same_cases": null, |
| "median_ms": 92.7, |
| "index_raw": 0.2936, |
| "index_skill": 0.2936, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "50": { |
| "catalog_id": 50, |
| "dataset": "Habermas Machine", |
| "requests": 1676, |
| "answered": 1676, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1676, |
| "metric": "accuracy", |
| "score": 0.4421, |
| "reference_same_cases": null, |
| "median_ms": 102.4, |
| "index_raw": 0.4421, |
| "index_skill": 0.1903, |
| "coverage": 1.0, |
| "chance": 0.311, |
| "in_index": true |
| }, |
| "56": { |
| "catalog_id": 56, |
| "dataset": "PhishNChips phishing decisions", |
| "requests": 2000, |
| "answered": 2000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.692, |
| "median_ms": 196.0, |
| "scored_requests": 2000, |
| "index_raw": 0.692, |
| "index_skill": 0.384, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "57": { |
| "catalog_id": 57, |
| "dataset": "MMLU-Pro", |
| "requests": 12032, |
| "answered": 12032, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.817, |
| "median_ms": 64.5, |
| "scored_requests": 12032, |
| "index_raw": 0.817, |
| "index_skill": 0.7942, |
| "coverage": 1.0, |
| "chance": 0.1109, |
| "in_index": true |
| }, |
| "58": { |
| "catalog_id": 58, |
| "dataset": "BBH fixed-option tasks", |
| "requests": 5507, |
| "answered": 5507, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.7954, |
| "median_ms": 64.0, |
| "scored_requests": 5507, |
| "index_raw": 0.7954, |
| "index_skill": 0.7034, |
| "coverage": 1.0, |
| "chance": 0.3101, |
| "in_index": true |
| }, |
| "59": { |
| "catalog_id": 59, |
| "dataset": "RAGTruth response-level hallucination", |
| "requests": 2700, |
| "answered": 2700, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "F1 on hallucinated class", |
| "score": 0.8405, |
| "median_ms": 95.3, |
| "scored_requests": 2700, |
| "index_raw": 0.8405, |
| "index_skill": 0.6693, |
| "coverage": 1.0, |
| "chance": 0.5177, |
| "in_index": true |
| }, |
| "61": { |
| "catalog_id": 61, |
| "dataset": "HoVer claim verification", |
| "requests": 4000, |
| "answered": 4000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.8632, |
| "median_ms": 75.6, |
| "scored_requests": 4000, |
| "index_raw": 0.8632, |
| "index_skill": 0.7264, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "62": { |
| "catalog_id": 62, |
| "dataset": "When2Call MCQ", |
| "requests": 3652, |
| "answered": 3652, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.8612, |
| "median_ms": 93.7, |
| "scored_requests": 3652, |
| "index_raw": 0.8612, |
| "index_skill": 0.8149, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "64": { |
| "catalog_id": 64, |
| "dataset": "New Yorker caption matching", |
| "requests": 528, |
| "answered": 528, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.7083, |
| "median_ms": 63.8, |
| "scored_requests": 528, |
| "index_raw": 0.7083, |
| "index_skill": 0.6354, |
| "coverage": 1.0, |
| "chance": 0.2, |
| "in_index": true |
| } |
| }, |
| "panel_id": "decision-index-0.2.1", |
| "note": "Decision Index 0.2.1 averages 38 benchmarks in five areas. Arts & Human Taste weighs 10%; the other four share 90% in proportion to the square root of their benchmark count (knowledge 25.8%, language 25.8%, retrieval 20.0%, tools 18.3%). Inside an area, gold ★ benchmarks weigh 1.2 and the rest 1.0; the index is 100 x the weighted mean of the five areas. Each benchmark is chance-corrected first, (score - chance) / (1 - chance) clipped to 0-1, so 0 means random guessing and 100 means perfect. Every score is coverage-adjusted, so an unanswered or unsupported request counts as wrong. ForecastBench enters against its baseline: clip((0.25 - Brier) / 0.25) x coverage, so always predicting 0.5 scores zero. MMLU, ARC-Easy, ARC-Challenge, RouterBench, SGD stay on the board as non-index benchmarks. The six interactive environments are still unrun and stay out. Every entrant on the board has results on all 38 index benchmarks. Point estimates only, no uncertainty intervals yet." |
| } |
|
|
