| { |
| "date": "2026-10-03", |
| "edition": "0.2.1", |
| "metric": "official_raw_score", |
| "code_revision": "87d4650b42b377c0291a89c1f1a879f9b31082bf", |
| "source_url": "https://github.com/apolinario/decision-index/tree/87d4650b42b377c0291a89c1f1a879f9b31082bf", |
| "jev_reference_url": "https://github.com/apolinario/decision-index/blob/87d4650b42b377c0291a89c1f1a879f9b31082bf/tests/fixtures/board-0.2.1.json", |
| "results": { |
| "gsm8k": { |
| "catalog_id": 30, |
| "title": "GSM8K", |
| "metric": "mean track accuracy", |
| "total": 2638, |
| "models": { |
| "qwen35_2b_base": { |
| "raw": 0.304, |
| "skill": 0.1609, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.30401819560272936, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.40788476118271416, |
| "skill": 0.21051301491028554, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.2001516300227445, |
| "skill": 0.11127958891416054, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_2b": { |
| "raw": 0.3776, |
| "skill": 0.2531, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.3775587566338135, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.5026535253980288, |
| "skill": 0.3368713671973717, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.25246398786959817, |
| "skill": 0.16940443096622018, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_4b_base": { |
| "raw": 0.37, |
| "skill": 0.2381, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.36997725549658833, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.44275966641395, |
| "skill": 0.2570128885519333, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.2971948445792267, |
| "skill": 0.2191053828658074, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_4b": { |
| "raw": 0.5459, |
| "skill": 0.4585, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.5458680818802123, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.6679302501895376, |
| "skill": 0.5572403335860501, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.42380591357088704, |
| "skill": 0.3597843484120967, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_9b_base": { |
| "raw": 0.5553, |
| "skill": 0.4642, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.5553449583017438, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.624715693707354, |
| "skill": 0.4996209249431387, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.48597422289613346, |
| "skill": 0.4288602476623705, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_9b": { |
| "raw": 0.6175, |
| "skill": 0.5442, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.6175132676269901, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.7225170583775588, |
| "skill": 0.6300227445034117, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.5125094768764216, |
| "skill": 0.458343863196024, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation", |
| "model_revision": "v0.3.0" |
| }, |
| "kev_9b": { |
| "raw": 0.4636, |
| "skill": 0.357, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.4636087945413192, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.5769522365428355, |
| "skill": 0.43593631539044725, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.3502653525398029, |
| "skill": 0.2780726139331143, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "kev_4b": { |
| "raw": 0.5849, |
| "skill": 0.5003, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.5849128127369219, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.6535253980288097, |
| "skill": 0.5380338640384129, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.5163002274450341, |
| "skill": 0.46255580827226017, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_pointer": { |
| "raw": 0.4761, |
| "skill": 0.3736, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.4761182714177407, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.601213040181956, |
| "skill": 0.46828405357594133, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.3510235026535254, |
| "skill": 0.2789150029483615, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_direct": { |
| "raw": 0.4503, |
| "skill": 0.3407, |
| "coverage": 1.0, |
| "answered": 2638, |
| "native_score": 0.45034116755117515, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "raw": 0.5625473843821076, |
| "skill": 0.41672984584281014, |
| "random": 0.25, |
| "coverage": 1 |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "raw": 0.33813495072024263, |
| "skill": 0.2645943896891585, |
| "random": 0.1, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jev": { |
| "raw": 0.7987, |
| "skill": 0.7565, |
| "coverage": 1, |
| "source": "published_leaderboard", |
| "version": "Jev 1.13.0" |
| } |
| } |
| }, |
| "chessbench": { |
| "catalog_id": 31, |
| "title": "ChessBench", |
| "metric": "best-move accuracy, accepting tied optima", |
| "total": 5000, |
| "models": { |
| "qwen35_2b_base": { |
| "raw": 0.0884, |
| "skill": 0.0071, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.0884, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.0884, |
| "skill": 0.00712784287150035, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_2b": { |
| "raw": 0.1098, |
| "skill": 0.0304, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1098, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1098, |
| "skill": 0.03043572369922071, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_4b_base": { |
| "raw": 0.1178, |
| "skill": 0.0391, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1178, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1178, |
| "skill": 0.0391489501768732, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_4b": { |
| "raw": 0.1242, |
| "skill": 0.0461, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1242, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1242, |
| "skill": 0.04611953135899518, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_9b_base": { |
| "raw": 0.1322, |
| "skill": 0.0548, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1322, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1322, |
| "skill": 0.054832757836647664, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_9b": { |
| "raw": 0.1034, |
| "skill": 0.0235, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1034, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1034, |
| "skill": 0.023465142517098744, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation", |
| "model_revision": "v0.3.0" |
| }, |
| "kev_9b": { |
| "raw": 0.1176, |
| "skill": 0.0389, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1176, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1176, |
| "skill": 0.038931119514931875, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "kev_4b": { |
| "raw": 0.088, |
| "skill": 0.0067, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.088, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.088, |
| "skill": 0.0066921815476177135, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_pointer": { |
| "raw": 0.1132, |
| "skill": 0.0341, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1132, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1132, |
| "skill": 0.03413884495222302, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_direct": { |
| "raw": 0.1172, |
| "skill": 0.0385, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.1172, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.1172, |
| "skill": 0.038495458191049256, |
| "random": 0.0818556110623024, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jev": { |
| "raw": 0.1722, |
| "skill": 0.0984, |
| "coverage": 1, |
| "source": "published_leaderboard", |
| "version": "Jev 1.13.0" |
| } |
| } |
| }, |
| "esci": { |
| "catalog_id": 37, |
| "title": "Amazon ESCI", |
| "metric": "macro-F1", |
| "total": 5000, |
| "models": { |
| "qwen35_2b_base": { |
| "raw": 0.281, |
| "skill": 0.0982, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.281041285029981, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.281041285029981, |
| "skill": 0.09821544691532137, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_2b": { |
| "raw": 0.3634, |
| "skill": 0.2015, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.3634004876980141, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.3634004876980141, |
| "skill": 0.2015179804599628, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_4b_base": { |
| "raw": 0.2918, |
| "skill": 0.1117, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.2918028420723953, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.2918028420723953, |
| "skill": 0.11171358763734261, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_4b": { |
| "raw": 0.4249, |
| "skill": 0.2786, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.4248617285215504, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.4248617285215504, |
| "skill": 0.2786083563522611, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_9b_base": { |
| "raw": 0.3056, |
| "skill": 0.129, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.3055619197954184, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.3055619197954184, |
| "skill": 0.12897149618892217, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_9b": { |
| "raw": 0.4708, |
| "skill": 0.3363, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.4708360372719961, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.4708360372719961, |
| "skill": 0.336273588870691, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation", |
| "model_revision": "v0.3.0" |
| }, |
| "kev_9b": { |
| "raw": 0.4809, |
| "skill": 0.3489, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.4809184305561307, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.4809184305561307, |
| "skill": 0.3489198595569521, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "kev_4b": { |
| "raw": 0.3956, |
| "skill": 0.2419, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.39561788161121436, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.39561788161121436, |
| "skill": 0.24192801731830993, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_pointer": { |
| "raw": 0.5494, |
| "skill": 0.4348, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.5493592773788667, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.5493592773788667, |
| "skill": 0.4347647031893874, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_direct": { |
| "raw": 0.5425, |
| "skill": 0.4262, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.5424928991743364, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.5424928991743364, |
| "skill": 0.42615225622747677, |
| "random": 0.20273782411694521, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jev": { |
| "raw": 0.5521, |
| "skill": 0.4382, |
| "coverage": 1, |
| "source": "published_leaderboard", |
| "version": "Jev 1.13.0" |
| } |
| } |
| }, |
| "bpomp": { |
| "catalog_id": 20, |
| "title": "BPoMP", |
| "metric": "mean accuracy across poem variants", |
| "total": 5000, |
| "models": { |
| "qwen35_2b_base": { |
| "raw": 0.505, |
| "skill": 0.0101, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.5026, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.5050301177080375, |
| "skill": 0.010060235416075036, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_2b": { |
| "raw": 0.7381, |
| "skill": 0.4763, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.7346, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.7381339972176438, |
| "skill": 0.47626799443528767, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_4b_base": { |
| "raw": 0.6895, |
| "skill": 0.3791, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.6942, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.689541386353874, |
| "skill": 0.37908277270774793, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_4b": { |
| "raw": 0.7819, |
| "skill": 0.5638, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.779, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.7818951793165829, |
| "skill": 0.5637903586331658, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qwen35_9b_base": { |
| "raw": 0.5939, |
| "skill": 0.1877, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.5954, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.593868528281851, |
| "skill": 0.187737056563702, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "qev_9b": { |
| "raw": 0.8103, |
| "skill": 0.6205, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.8068, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.8102618917009247, |
| "skill": 0.6205237834018493, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation", |
| "model_revision": "v0.3.0" |
| }, |
| "kev_9b": { |
| "raw": 0.6693, |
| "skill": 0.3387, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.6688, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.6693391049629296, |
| "skill": 0.3386782099258592, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "kev_4b": { |
| "raw": 0.6528, |
| "skill": 0.3056, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.6502, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.6528226564034934, |
| "skill": 0.3056453128069867, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_pointer": { |
| "raw": 0.7387, |
| "skill": 0.4773, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.731, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.7386594480630367, |
| "skill": 0.4773188961260735, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jevany_4b_direct": { |
| "raw": 0.8129, |
| "skill": 0.6258, |
| "coverage": 1.0, |
| "answered": 5000, |
| "native_score": 0.8142, |
| "tracks": [ |
| { |
| "track": "all", |
| "raw": 0.8128861556266215, |
| "skill": 0.625772311253243, |
| "random": 0.5, |
| "coverage": 1 |
| } |
| ], |
| "source": "local_evaluation" |
| }, |
| "jev": { |
| "raw": 0.9092, |
| "skill": 0.8184, |
| "coverage": 1, |
| "source": "published_leaderboard", |
| "version": "Jev 1.13.0" |
| } |
| } |
| } |
| } |
| } |
|
|
