Buckets:
| { | |
| "config": { | |
| "n_boot": 10000, | |
| "seed": 20261101, | |
| "ci": "percentile 95%", | |
| "eval_items": 5442, | |
| "thinking_items": 1498, | |
| "in_dist_items": 1125, | |
| "ood_items": 4317, | |
| "calib_items": 515, | |
| "s1mb_benchmarks": 137, | |
| "weighted_stat_check_max_abs_err": 8.326672684688674e-17, | |
| "eval_file": "runs/v2/data/eval.jsonl", | |
| "calib_file": "runs/v2/data/calib.jsonl (identical to data/calib.jsonl)", | |
| "readout_files": { | |
| "base": "results/predictions/readout_base.jsonl", | |
| "v1": "results/predictions/readout_joint.jsonl", | |
| "v2": "runs/v2/results/predictions/readout_v2_joint.jsonl", | |
| "rl60": "runs/v2/results/predictions/readout_rl60.jsonl", | |
| "rl40": "runs/v2/results/predictions/readout_rl40.jsonl", | |
| "abl_gold": "runs/v2/results/predictions/readout_abl_gold.jsonl", | |
| "abl_mix": "runs/v2/results/predictions/readout_abl_mix.jsonl" | |
| }, | |
| "reason_files": { | |
| "base": "results/predictions/reason_base.jsonl", | |
| "v1": "results/predictions/reason_joint.jsonl", | |
| "v2": "runs/v2/results/predictions/reason_v2_joint.jsonl", | |
| "rl60": "runs/v2/results/predictions/reason_rl60.jsonl", | |
| "rl40": "runs/v2/results/predictions/reason_rl40.jsonl" | |
| }, | |
| "temperatures": { | |
| "base": { | |
| "choice": 1.707683209388825, | |
| "noul": 2.4105066990845945, | |
| "score": 1.2847580560947693 | |
| }, | |
| "v1": { | |
| "choice": 1.2843740495491411, | |
| "noul": 1.1776347713530488, | |
| "score": 1.0535480782391846 | |
| }, | |
| "v2": { | |
| "choice": 1.1592816013856364, | |
| "noul": 0.8031824332990728, | |
| "score": 0.716891860990727 | |
| }, | |
| "rl60": { | |
| "choice": 1.1599743376333564, | |
| "noul": 0.7983784191901301, | |
| "score": 0.7276446912498098 | |
| }, | |
| "rl40": { | |
| "choice": 1.1597719952546015, | |
| "noul": 0.8079639789416141, | |
| "score": 0.7152072687033781 | |
| }, | |
| "abl_gold": { | |
| "choice": 1.0763093956254552, | |
| "noul": 0.9000382248702841, | |
| "score": 0.9022968292868349 | |
| }, | |
| "abl_mix": { | |
| "choice": 1.1713673523344856, | |
| "noul": 0.8390986564915838, | |
| "score": 0.8453427579160532 | |
| } | |
| } | |
| }, | |
| "reproduction": { | |
| "all_match": true, | |
| "mismatches": [], | |
| "rows": [ | |
| { | |
| "what": "fast readout accuracy, all 5,442: base", | |
| "recomputed": 62.66078647556046, | |
| "metrics_json": 62.66078647556046, | |
| "matches_json": true, | |
| "final_report": 62.7, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout ECE15: base", | |
| "recomputed": 0.03566369525322492, | |
| "metrics_json": 0.03566369525322492, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout accuracy, all 5,442: v1", | |
| "recomputed": 69.45975744211687, | |
| "metrics_json": 69.45975744211687, | |
| "matches_json": true, | |
| "final_report": 69.5, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout ECE15: v1", | |
| "recomputed": 0.030361916222813444, | |
| "metrics_json": 0.030361916222813444, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout accuracy, all 5,442: v2", | |
| "recomputed": 72.6938625505329, | |
| "metrics_json": 72.6938625505329, | |
| "matches_json": true, | |
| "final_report": 72.7, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout ECE15: v2", | |
| "recomputed": 0.032551733104682894, | |
| "metrics_json": 0.032551733104682894, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout accuracy, all 5,442: rl60", | |
| "recomputed": 72.45497978684308, | |
| "metrics_json": 72.45497978684308, | |
| "matches_json": true, | |
| "final_report": 72.5, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout ECE15: rl60", | |
| "recomputed": 0.029896171411952138, | |
| "metrics_json": 0.029896171411952138, | |
| "matches_json": true, | |
| "final_report": 0.03, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout accuracy, all 5,442: rl40", | |
| "recomputed": 72.74898934215362, | |
| "metrics_json": 72.74898934215362, | |
| "matches_json": true, | |
| "final_report": 72.7, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout ECE15: rl40", | |
| "recomputed": 0.031382864409026635, | |
| "metrics_json": 0.031382864409026635, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout accuracy, all 5,442: abl_gold", | |
| "recomputed": 71.7015803013598, | |
| "metrics_json": 71.7015803013598, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout ECE15: abl_gold", | |
| "recomputed": 0.030737215213212113, | |
| "metrics_json": 0.030737215213212113, | |
| "matches_json": true, | |
| "final_report": 0.031, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "ablation readout accuracy: abl_gold", | |
| "recomputed": 71.7015803013598, | |
| "final_report": 71.7, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout accuracy, all 5,442: abl_mix", | |
| "recomputed": 70.5071664829107, | |
| "metrics_json": 70.5071664829107, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout ECE15: abl_mix", | |
| "recomputed": 0.04687347907023101, | |
| "metrics_json": 0.04687347907023101, | |
| "matches_json": true, | |
| "final_report": 0.047, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "ablation readout accuracy: abl_mix", | |
| "recomputed": 70.5071664829107, | |
| "final_report": 70.5, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "thinking accuracy, 1,498: base", | |
| "recomputed": 84.04539385847796, | |
| "metrics_json": 84.04539385847796, | |
| "matches_json": true, | |
| "final_report": 84.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "looped chains %: base", | |
| "recomputed": 0.8678237650200267, | |
| "metrics_json": 0.8678237650200267, | |
| "matches_json": true, | |
| "final_report": 0.9, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "post-chain ECE15: base", | |
| "recomputed": 0.049625611476746045, | |
| "metrics_json": 0.049625611476746045, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout accuracy on the 1,498 subset: base", | |
| "recomputed": 61.815754339118826, | |
| "metrics_json": 61.815754339118826, | |
| "matches_json": true, | |
| "final_report": 61.8, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% accuracy: base", | |
| "recomputed": 73.03070761014686, | |
| "metrics_json": 73.03070761014686, | |
| "matches_json": true, | |
| "final_report": 73.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% thinking tokens per question: base", | |
| "recomputed": 1069.6748998664887, | |
| "metrics_json": 1069.6748998664887, | |
| "matches_json": true, | |
| "final_report": 1069, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "gap recovered at f=0.3: base", | |
| "recomputed": 0.495575221238938, | |
| "metrics_json": 0.495575221238938, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "disagreement AUROC: base", | |
| "recomputed": 0.7728567119155354, | |
| "metrics_json": 0.7728567119155354, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "thinking accuracy, 1,498: v1", | |
| "recomputed": 84.37917222963952, | |
| "metrics_json": 84.37917222963952, | |
| "matches_json": true, | |
| "final_report": 84.4, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "looped chains %: v1", | |
| "recomputed": 1.4018691588785046, | |
| "metrics_json": 1.4018691588785046, | |
| "matches_json": true, | |
| "final_report": 1.4, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "post-chain ECE15: v1", | |
| "recomputed": 0.022557188016222515, | |
| "metrics_json": 0.022557188016222515, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout accuracy on the 1,498 subset: v1", | |
| "recomputed": 70.29372496662216, | |
| "metrics_json": 70.29372496662216, | |
| "matches_json": true, | |
| "final_report": 70.3, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% accuracy: v1", | |
| "recomputed": 77.97062750333778, | |
| "metrics_json": 77.97062750333778, | |
| "matches_json": true, | |
| "final_report": 78.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% thinking tokens per question: v1", | |
| "recomputed": 1166.328437917223, | |
| "metrics_json": 1166.328437917223, | |
| "matches_json": true, | |
| "final_report": 1166, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "gap recovered at f=0.3: v1", | |
| "recomputed": 0.5180180180180183, | |
| "metrics_json": 0.5180180180180183, | |
| "matches_json": true, | |
| "final_report": 0.52, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "disagreement AUROC: v1", | |
| "recomputed": 0.7796652398582825, | |
| "metrics_json": 0.7796652398582825, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "thinking accuracy, 1,498: v2", | |
| "recomputed": 81.17489986648864, | |
| "metrics_json": 81.17489986648864, | |
| "matches_json": true, | |
| "final_report": 81.2, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "looped chains %: v2", | |
| "recomputed": 5.07343124165554, | |
| "metrics_json": 5.07343124165554, | |
| "matches_json": true, | |
| "final_report": 5.1, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "post-chain ECE15: v2", | |
| "recomputed": 0.04318024724951665, | |
| "metrics_json": 0.04318024724951665, | |
| "matches_json": true, | |
| "final_report": 0.043, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout accuracy on the 1,498 subset: v2", | |
| "recomputed": 75.9679572763685, | |
| "metrics_json": 75.9679572763685, | |
| "matches_json": true, | |
| "final_report": 76.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% accuracy: v2", | |
| "recomputed": 81.77570093457945, | |
| "metrics_json": 81.77570093457945, | |
| "matches_json": true, | |
| "final_report": 81.8, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% thinking tokens per question: v2", | |
| "recomputed": 1749.4352469959947, | |
| "metrics_json": 1749.4352469959947, | |
| "matches_json": true, | |
| "final_report": 1749, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "gap recovered at f=0.3: v2", | |
| "recomputed": 0.7016129032258068, | |
| "metrics_json": 0.7016129032258068, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "disagreement AUROC: v2", | |
| "recomputed": 0.8387407027442934, | |
| "metrics_json": 0.8387407027442934, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "thinking accuracy, 1,498: rl60", | |
| "recomputed": 83.11081441922563, | |
| "metrics_json": 83.11081441922563, | |
| "matches_json": true, | |
| "final_report": 83.1, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "looped chains %: rl60", | |
| "recomputed": 0.20026702269692925, | |
| "metrics_json": 0.20026702269692925, | |
| "matches_json": true, | |
| "final_report": 0.2, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "post-chain ECE15: rl60", | |
| "recomputed": 0.056783666219741305, | |
| "metrics_json": 0.056783666219741305, | |
| "matches_json": true, | |
| "final_report": 0.057, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "fast readout accuracy on the 1,498 subset: rl60", | |
| "recomputed": 75.90120160213618, | |
| "metrics_json": 75.90120160213618, | |
| "matches_json": true, | |
| "final_report": 75.9, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% accuracy: rl60", | |
| "recomputed": 81.84245660881176, | |
| "metrics_json": 81.84245660881176, | |
| "matches_json": true, | |
| "final_report": 81.8, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% thinking tokens per question: rl60", | |
| "recomputed": 729.0380507343124, | |
| "metrics_json": 729.0380507343124, | |
| "matches_json": true, | |
| "final_report": 729, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "gap recovered at f=0.3: rl60", | |
| "recomputed": 0.7946428571428577, | |
| "metrics_json": 0.7946428571428577, | |
| "matches_json": true, | |
| "final_report": 0.79, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "disagreement AUROC: rl60", | |
| "recomputed": 0.8328178476359488, | |
| "metrics_json": 0.8328178476359488, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "thinking accuracy, 1,498: rl40", | |
| "recomputed": 83.5781041388518, | |
| "metrics_json": 83.5781041388518, | |
| "matches_json": true, | |
| "final_report": 83.6, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "looped chains %: rl40", | |
| "recomputed": 2.0026702269692924, | |
| "metrics_json": 2.0026702269692924, | |
| "matches_json": true, | |
| "final_report": 2.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "post-chain ECE15: rl40", | |
| "recomputed": 0.05562235339316634, | |
| "metrics_json": 0.05562235339316634, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "fast readout accuracy on the 1,498 subset: rl40", | |
| "recomputed": 75.9679572763685, | |
| "metrics_json": 75.9679572763685, | |
| "matches_json": true, | |
| "final_report": 76.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% accuracy: rl40", | |
| "recomputed": 82.04272363150868, | |
| "metrics_json": 82.04272363150868, | |
| "matches_json": true, | |
| "final_report": 82.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "routed 30% thinking tokens per question: rl40", | |
| "recomputed": 1079.0353805073432, | |
| "metrics_json": 1079.0353805073432, | |
| "matches_json": true, | |
| "final_report": 1079, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "gap recovered at f=0.3: rl40", | |
| "recomputed": 0.67910447761194, | |
| "metrics_json": 0.67910447761194, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "disagreement AUROC: rl40", | |
| "recomputed": 0.8391960206975687, | |
| "metrics_json": 0.8391960206975687, | |
| "matches_json": true | |
| }, | |
| { | |
| "what": "S1MB Task Avg: rl60", | |
| "recomputed": 47.88785779401238, | |
| "metrics_json": 47.88785779401238, | |
| "matches_json": true, | |
| "final_report": 47.9, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB Task Avg: v1_joint", | |
| "recomputed": 43.45462995779747, | |
| "metrics_json": 43.45462995779747, | |
| "matches_json": true, | |
| "final_report": 43.5, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB Task Avg: v2_2b_ro", | |
| "recomputed": 37.29524401796516, | |
| "metrics_json": 37.29524401796516, | |
| "matches_json": true, | |
| "final_report": 37.3, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB Task Avg: base", | |
| "recomputed": 32.78395182793917, | |
| "metrics_json": 32.78395182793917, | |
| "matches_json": true, | |
| "final_report": 32.8, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB Task Avg: base_2b", | |
| "recomputed": 18.13781304391087, | |
| "metrics_json": 18.13781304391087, | |
| "matches_json": true, | |
| "final_report": 18.1, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB seen mean adjusted: base", | |
| "recomputed": 44.45745951641978, | |
| "final_report": 44.5, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB seen mean adjusted: rl60", | |
| "recomputed": 52.81161014738631, | |
| "final_report": 52.8, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB unseen mean adjusted: base", | |
| "recomputed": 33.0320179498522, | |
| "final_report": 33.0, | |
| "matches_report": true | |
| }, | |
| { | |
| "what": "S1MB unseen mean adjusted: rl60", | |
| "recomputed": 50.38894667156569, | |
| "final_report": 50.4, | |
| "matches_report": true | |
| } | |
| ] | |
| }, | |
| "fast_readout": { | |
| "acc_base": { | |
| "point": 62.66078647556046, | |
| "ci95": [ | |
| 61.392870268283716, | |
| 63.96545387725101 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_v1": { | |
| "point": 69.45975744211687, | |
| "ci95": [ | |
| 68.24650863653069, | |
| 70.65417126056597 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_v2": { | |
| "point": 72.6938625505329, | |
| "ci95": [ | |
| 71.48107313487688, | |
| 73.8699007717751 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_rl60": { | |
| "point": 72.45497978684308, | |
| "ci95": [ | |
| 71.27894156560089, | |
| 73.63101800808526 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_rl40": { | |
| "point": 72.74898934215362, | |
| "ci95": [ | |
| 71.55457552370453, | |
| 73.9250275633958 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_abl_gold": { | |
| "point": 71.7015803013598, | |
| "ci95": [ | |
| 70.5071664829107, | |
| 72.9143697170158 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_abl_mix": { | |
| "point": 70.5071664829107, | |
| "ci95": [ | |
| 69.27600147004777, | |
| 71.73833149577361 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "ece_base": { | |
| "point": 0.03566369525322493, | |
| "ci95": [ | |
| 0.027999627125269128, | |
| 0.04845499652988414 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "ece_v1": { | |
| "point": 0.03036191622281344, | |
| "ci95": [ | |
| 0.025026237714199515, | |
| 0.043861464771344234 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "ece_v2": { | |
| "point": 0.032551733104682845, | |
| "ci95": [ | |
| 0.025758067703432724, | |
| 0.04462291902602932 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "ece_rl60": { | |
| "point": 0.029896171411952062, | |
| "ci95": [ | |
| 0.023890965253341733, | |
| 0.04236846421496938 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "ece_rl40": { | |
| "point": 0.03138286440902665, | |
| "ci95": [ | |
| 0.025889901115521703, | |
| 0.04458527559671286 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "ece_abl_gold": { | |
| "point": 0.03073721521321206, | |
| "ci95": [ | |
| 0.024175667222425857, | |
| 0.04285190233471198 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "ece_abl_mix": { | |
| "point": 0.046873479070231004, | |
| "ci95": [ | |
| 0.03888822820449279, | |
| 0.059150944385858946 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_rl60-base": { | |
| "point": 9.794193311282617, | |
| "ci95": [ | |
| 8.434399117971337, | |
| 11.153987504593898 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_rl60-v1": { | |
| "point": 2.9952223447262116, | |
| "ci95": [ | |
| 1.8738515251745655, | |
| 4.152884968761484 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_v2-v1": { | |
| "point": 3.23410510841603, | |
| "ci95": [ | |
| 2.113193678794556, | |
| 4.391767732451313 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_rl60-v2": { | |
| "point": -0.2388827636898183, | |
| "ci95": [ | |
| -0.4961411245865577, | |
| 0.018375597206909955 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.9678, | |
| "n_boot": 10000 | |
| }, | |
| "acc_rl40-rl60": { | |
| "point": 0.2940095553105482, | |
| "ci95": [ | |
| 0.03675119441381991, | |
| 0.5512679162072764 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0111, | |
| "n_boot": 10000 | |
| }, | |
| "acc_abl_mix-abl_gold": { | |
| "point": -1.1944138184490916, | |
| "ci95": [ | |
| -2.131569276001477, | |
| -0.2756339581036382 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.9951, | |
| "n_boot": 10000 | |
| }, | |
| "ece_abl_mix-abl_gold": { | |
| "point": 0.016136263857018943, | |
| "ci95": [ | |
| 0.006223402671340885, | |
| 0.02535460432113202 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0007, | |
| "n_boot": 10000 | |
| }, | |
| "ece_rl60-v2": { | |
| "point": -0.002655561692730783, | |
| "ci95": [ | |
| -0.008835509126862151, | |
| 0.003929426103087335 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.7264, | |
| "n_boot": 10000 | |
| } | |
| }, | |
| "in_distribution": { | |
| "acc_abl_mix": { | |
| "point": 77.86666666666666, | |
| "ci95": [ | |
| 75.37777777777778, | |
| 80.26666666666667 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_abl_gold": { | |
| "point": 76.35555555555555, | |
| "ci95": [ | |
| 73.86666666666667, | |
| 78.84444444444443 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_abl_mix-abl_gold": { | |
| "point": 1.5111111111111075, | |
| "ci95": [ | |
| -0.4444444444444362, | |
| 3.3777777777777795 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.0651, | |
| "n_boot": 10000 | |
| }, | |
| "in_minus_ood_acc_abl_mix-abl_gold": { | |
| "point": 3.410578333719405, | |
| "ci95": [ | |
| 1.2282464674542508, | |
| 5.535263686201841 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0011, | |
| "n_boot": 10000 | |
| } | |
| }, | |
| "out_of_distribution": { | |
| "acc_abl_mix": { | |
| "point": 68.589298123697, | |
| "ci95": [ | |
| 67.22260829279593, | |
| 70.0023164234422 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_abl_gold": { | |
| "point": 70.4887653463053, | |
| "ci95": [ | |
| 69.14523974982627, | |
| 71.8554551772064 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "acc_abl_mix-abl_gold": { | |
| "point": -1.8994672226082976, | |
| "ci95": [ | |
| -2.9418577716006533, | |
| -0.8570766736159419 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.9998, | |
| "n_boot": 10000 | |
| } | |
| }, | |
| "thinking_subset": { | |
| "think_base": { | |
| "point": 84.04539385847796, | |
| "ci95": [ | |
| 82.1762349799733, | |
| 85.84779706275033 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "loop_base": { | |
| "point": 0.8678237650200267, | |
| "ci95": [ | |
| 0.46728971962616817, | |
| 1.4018691588785046 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "postece_base": { | |
| "point": 0.049625611476746045, | |
| "ci95": [ | |
| 0.038016989461121524, | |
| 0.06919877581458238 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_base": { | |
| "point": 73.03070761014686, | |
| "ci95": [ | |
| 70.89452603471294, | |
| 75.56742323097463 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "tokens30_base": { | |
| "point": 1069.6748998664887, | |
| "ci95": [ | |
| 972.5753170894526, | |
| 1171.397096128171 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "gap_base": { | |
| "point": 0.495575221238938, | |
| "ci95": [ | |
| 0.4492288135593217, | |
| 0.5634927097874978 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "disauroc_base": { | |
| "point": 0.7728567119155354, | |
| "ci95": [ | |
| 0.7491228253788498, | |
| 0.79620098540328 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "fastsub_base": { | |
| "point": 61.815754339118826, | |
| "ci95": [ | |
| 59.345794392523366, | |
| 64.28571428571429 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "think_v1": { | |
| "point": 84.37917222963952, | |
| "ci95": [ | |
| 82.51001335113484, | |
| 86.18157543391189 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "loop_v1": { | |
| "point": 1.4018691588785046, | |
| "ci95": [ | |
| 0.8678237650200267, | |
| 2.0026702269692924 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "postece_v1": { | |
| "point": 0.022557188016222536, | |
| "ci95": [ | |
| 0.012591878458476684, | |
| 0.04263529896371238 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_v1": { | |
| "point": 77.97062750333778, | |
| "ci95": [ | |
| 75.90120160213618, | |
| 80.37383177570094 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "tokens30_v1": { | |
| "point": 1166.328437917223, | |
| "ci95": [ | |
| 1025.3066255006675, | |
| 1312.2899365821095 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "gap_v1": { | |
| "point": 0.5180180180180183, | |
| "ci95": [ | |
| 0.4517725560831827, | |
| 0.6093023255813952 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "disauroc_v1": { | |
| "point": 0.7796652398582825, | |
| "ci95": [ | |
| 0.7542981427603564, | |
| 0.8044390460296513 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "fastsub_v1": { | |
| "point": 70.29372496662216, | |
| "ci95": [ | |
| 67.89052069425901, | |
| 72.5634178905207 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "think_v2": { | |
| "point": 81.17489986648864, | |
| "ci95": [ | |
| 79.17222963951936, | |
| 83.11081441922563 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "loop_v2": { | |
| "point": 5.07343124165554, | |
| "ci95": [ | |
| 4.005340453938585, | |
| 6.208277703604806 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "postece_v2": { | |
| "point": 0.043180247249516655, | |
| "ci95": [ | |
| 0.03324441610409772, | |
| 0.0633360126176291 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_v2": { | |
| "point": 81.77570093457945, | |
| "ci95": [ | |
| 79.83978638184246, | |
| 83.77837116154873 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "tokens30_v2": { | |
| "point": 1749.4352469959947, | |
| "ci95": [ | |
| 1514.9725801068091, | |
| 2000.0111648865154 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "gap_v2": { | |
| "point": 0.7016129032258068, | |
| "ci95": [ | |
| 0.5815561603166747, | |
| 0.8514870823019801 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "disauroc_v2": { | |
| "point": 0.8387407027442934, | |
| "ci95": [ | |
| 0.815723469996804, | |
| 0.8604893093495404 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "fastsub_v2": { | |
| "point": 75.9679572763685, | |
| "ci95": [ | |
| 73.76502002670226, | |
| 78.1041388518024 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "think_rl60": { | |
| "point": 83.11081441922563, | |
| "ci95": [ | |
| 81.17489986648864, | |
| 84.97997329773031 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "loop_rl60": { | |
| "point": 0.20026702269692925, | |
| "ci95": [ | |
| 0.0, | |
| 0.46728971962616817 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.0437, | |
| "n_boot": 10000 | |
| }, | |
| "postece_rl60": { | |
| "point": 0.05678366621974126, | |
| "ci95": [ | |
| 0.04329510783104505, | |
| 0.07533556745011868 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_rl60": { | |
| "point": 81.84245660881176, | |
| "ci95": [ | |
| 79.77303070761015, | |
| 83.77837116154873 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "tokens30_rl60": { | |
| "point": 729.0380507343124, | |
| "ci95": [ | |
| 679.0986648865153, | |
| 780.0121662216288 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "gap_rl60": { | |
| "point": 0.7946428571428577, | |
| "ci95": [ | |
| 0.6399999999999991, | |
| 0.9726180283757333 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "disauroc_rl60": { | |
| "point": 0.8328178476359488, | |
| "ci95": [ | |
| 0.8107951744840382, | |
| 0.8532332355165317 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "fastsub_rl60": { | |
| "point": 75.90120160213618, | |
| "ci95": [ | |
| 73.69826435246995, | |
| 78.03738317757009 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "think_rl40": { | |
| "point": 83.5781041388518, | |
| "ci95": [ | |
| 81.64218958611482, | |
| 85.44726301735648 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "loop_rl40": { | |
| "point": 2.0026702269692924, | |
| "ci95": [ | |
| 1.335113484646195, | |
| 2.7369826435246996 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "postece_rl40": { | |
| "point": 0.05562235339316637, | |
| "ci95": [ | |
| 0.041059990175372915, | |
| 0.07261064975458964 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_rl40": { | |
| "point": 82.04272363150868, | |
| "ci95": [ | |
| 79.83978638184246, | |
| 83.97863818424565 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "tokens30_rl40": { | |
| "point": 1079.0353805073432, | |
| "ci95": [ | |
| 936.8369158878504, | |
| 1237.3037049399197 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "gap_rl40": { | |
| "point": 0.67910447761194, | |
| "ci95": [ | |
| 0.5508474576271178, | |
| 0.79381443298969 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "disauroc_rl40": { | |
| "point": 0.8391960206975687, | |
| "ci95": [ | |
| 0.8171549415586951, | |
| 0.8599649812340994 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "fastsub_rl40": { | |
| "point": 75.9679572763685, | |
| "ci95": [ | |
| 73.76502002670226, | |
| 78.1041388518024 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "think_rl60-v2": { | |
| "point": 1.9359145527369837, | |
| "ci95": [ | |
| 0.33377837116155273, | |
| 3.5380507343124257 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0095, | |
| "n_boot": 10000 | |
| }, | |
| "think_rl40-rl60": { | |
| "point": 0.4672897196261738, | |
| "ci95": [ | |
| -1.0680907877169576, | |
| 2.002670226969294 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.2931, | |
| "n_boot": 10000 | |
| }, | |
| "think_rl60-base": { | |
| "point": -0.9345794392523366, | |
| "ci95": [ | |
| -2.603471295060078, | |
| 0.7343124165554049 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.8674, | |
| "n_boot": 10000 | |
| }, | |
| "think_v2-base": { | |
| "point": -2.87049399198932, | |
| "ci95": [ | |
| -4.672897196261683, | |
| -1.001335113484647 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.9987, | |
| "n_boot": 10000 | |
| }, | |
| "think_rl40-base": { | |
| "point": -0.4672897196261627, | |
| "ci95": [ | |
| -2.202937249666226, | |
| 1.3351134846461887 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.7073, | |
| "n_boot": 10000 | |
| }, | |
| "loop_rl60-v2": { | |
| "point": -4.873164218958611, | |
| "ci95": [ | |
| -6.008010680907877, | |
| -3.8050734312416554 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 1.0, | |
| "n_boot": 10000 | |
| }, | |
| "loop_rl60-base": { | |
| "point": -0.6675567423230975, | |
| "ci95": [ | |
| -1.2016021361815754, | |
| -0.2002670226969292 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.9987, | |
| "n_boot": 10000 | |
| }, | |
| "postece_rl60-v2": { | |
| "point": 0.013603418970224608, | |
| "ci95": [ | |
| -0.005782815540674307, | |
| 0.028014653285939252 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.1016, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_rl60-v2": { | |
| "point": 0.06675567423231055, | |
| "ci95": [ | |
| -1.2683578104138782, | |
| 1.1348464619492682 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.5548, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_rl60-v1": { | |
| "point": 3.8718291054739673, | |
| "ci95": [ | |
| 1.6688918558077526, | |
| 5.607476635514019 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0001, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_rl60-base": { | |
| "point": 8.811748998664893, | |
| "ci95": [ | |
| 6.275033377837125, | |
| 10.747663551401876 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "routed30_rl40-rl60": { | |
| "point": 0.20026702269692054, | |
| "ci95": [ | |
| -1.0680907877169576, | |
| 1.4018691588784993 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.4054, | |
| "n_boot": 10000 | |
| }, | |
| "tokens30_rl60-v2": { | |
| "point": -1020.3971962616823, | |
| "ci95": [ | |
| -1262.4076435246996, | |
| -796.9301401869159 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 1.0, | |
| "n_boot": 10000 | |
| }, | |
| "tokens30_rl60-v1": { | |
| "point": -437.29038718291065, | |
| "ci95": [ | |
| -583.7585614152202, | |
| -295.69749666221634 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 1.0, | |
| "n_boot": 10000 | |
| }, | |
| "gap_rl60-v1": { | |
| "point": 0.2766248391248394, | |
| "ci95": [ | |
| 0.10222153827368344, | |
| 0.4416623395127453 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0007, | |
| "n_boot": 10000 | |
| }, | |
| "disauroc_rl60-base": { | |
| "point": 0.059961135720413394, | |
| "ci95": [ | |
| 0.03013448403970405, | |
| 0.08972870881776687 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000, | |
| "ok": true, | |
| "ci_lower_ge_0.03": true | |
| }, | |
| "disauroc_rl60-v1": { | |
| "point": 0.05315260777766628, | |
| "ci95": [ | |
| 0.026016003868889737, | |
| 0.0800455701108224 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0002, | |
| "n_boot": 10000, | |
| "ok": true, | |
| "ci_lower_ge_0.03": false | |
| }, | |
| "disauroc_v2-base": { | |
| "point": 0.06588399082875795, | |
| "ci95": [ | |
| 0.03548144843090986, | |
| 0.09598063336036833 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000, | |
| "ok": true, | |
| "ci_lower_ge_0.03": true | |
| } | |
| }, | |
| "s1mb": { | |
| "task_avg_rl60-base": { | |
| "point": 15.103905966073224, | |
| "ci95": [ | |
| 12.187213031565449, | |
| 18.109350969363067 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "noul_rl60-base": { | |
| "point": 16.936647603588582, | |
| "ci95": [ | |
| 11.088346187529707, | |
| 23.424695196328337 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "choice_rl60-base": { | |
| "point": 11.490184676531786, | |
| "ci95": [ | |
| 8.154788425174218, | |
| 15.18035665504268 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "score_rl60-base": { | |
| "point": 16.8848856180993, | |
| "ci95": [ | |
| 11.444263955871595, | |
| 22.20659192965336 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "task_avg_rl60-v1_joint": { | |
| "point": 4.43322783621492, | |
| "ci95": [ | |
| 2.382365569178491, | |
| 6.473912759006311 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "noul_rl60-v1_joint": { | |
| "point": 0.6240932048510955, | |
| "ci95": [ | |
| -2.669141362389978, | |
| 3.5847784365428166 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.3464, | |
| "n_boot": 10000 | |
| }, | |
| "choice_rl60-v1_joint": { | |
| "point": 4.883496187822424, | |
| "ci95": [ | |
| 1.429800587443443, | |
| 8.476671370052959 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0024, | |
| "n_boot": 10000 | |
| }, | |
| "score_rl60-v1_joint": { | |
| "point": 7.79209411597124, | |
| "ci95": [ | |
| 3.9107877421907564, | |
| 11.778298142115244 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "task_avg_v2_2b_ro-base_2b": { | |
| "point": 19.157430974054293, | |
| "ci95": [ | |
| 15.524056659215809, | |
| 22.84953402878724 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "noul_v2_2b_ro-base_2b": { | |
| "point": 19.381170191622655, | |
| "ci95": [ | |
| 13.501973831331975, | |
| 25.764464431181388 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "choice_v2_2b_ro-base_2b": { | |
| "point": 16.15072747725473, | |
| "ci95": [ | |
| 11.549157778631633, | |
| 20.962310902598073 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "score_v2_2b_ro-base_2b": { | |
| "point": 21.940395253285498, | |
| "ci95": [ | |
| 14.09623663233703, | |
| 29.705523652838984 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "task_avg_v1_joint-base": { | |
| "point": 10.670678129858302, | |
| "ci95": [ | |
| 8.098304378970248, | |
| 13.375483915108177 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "noul_v1_joint-base": { | |
| "point": 16.31255439873749, | |
| "ci95": [ | |
| 10.900416203693867, | |
| 22.412829304668776 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "choice_v1_joint-base": { | |
| "point": 6.606688488709363, | |
| "ci95": [ | |
| 3.289714822261716, | |
| 9.947506948034935 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0001, | |
| "n_boot": 10000 | |
| }, | |
| "score_v1_joint-base": { | |
| "point": 9.092791502128055, | |
| "ci95": [ | |
| 5.130706821300408, | |
| 13.335092056392126 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "seen_mean_base": { | |
| "point": 44.45745951641978, | |
| "ci95": [ | |
| 35.8722837140146, | |
| 53.12026620176346 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "seen_mean_rl60": { | |
| "point": 52.81161014738631, | |
| "ci95": [ | |
| 44.66679859753839, | |
| 60.62175585046779 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "seen_gain": { | |
| "point": 8.354150630966538, | |
| "ci95": [ | |
| 4.673487628754129, | |
| 12.19489432325701 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "unseen_mean_base": { | |
| "point": 33.0320179498522, | |
| "ci95": [ | |
| 27.220467181824816, | |
| 39.01885232148167 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "unseen_mean_rl60": { | |
| "point": 50.38894667156569, | |
| "ci95": [ | |
| 44.5865045804265, | |
| 56.25110344606372 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "unseen_gain": { | |
| "point": 17.356928721713498, | |
| "ci95": [ | |
| 13.422585908351362, | |
| 21.522589100576134 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000 | |
| }, | |
| "unseen_minus_seen_gain": { | |
| "point": 9.00277809074696, | |
| "ci95": [ | |
| 3.44111690419792, | |
| 14.502648038307575 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0006, | |
| "n_boot": 10000 | |
| }, | |
| "unseen_minus_seen_gain_noul_only": { | |
| "point": 11.014496516334948, | |
| "ci95": [ | |
| -0.8581084729523564, | |
| 22.568024672416126 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.0332, | |
| "n_boot": 10000, | |
| "n_seen": 15, | |
| "n_unseen": 44 | |
| }, | |
| "unseen_minus_seen_gain_choice_only": { | |
| "point": 5.361400410100033, | |
| "ci95": [ | |
| -0.8724300081687891, | |
| 11.907982236032042 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.0468, | |
| "n_boot": 10000, | |
| "n_seen": 25, | |
| "n_unseen": 32 | |
| }, | |
| "unseen_minus_seen_gain_task_matched": { | |
| "point": 8.18794846321749, | |
| "ci95": [ | |
| 1.5024181484550834, | |
| 14.72715700647714 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0082, | |
| "n_boot": 10000 | |
| }, | |
| "split_counts": { | |
| "seen": { | |
| "noul": 15, | |
| "choice": 25, | |
| "score": 1 | |
| }, | |
| "unseen": { | |
| "noul": 44, | |
| "choice": 32, | |
| "score": 20 | |
| } | |
| }, | |
| "seen_benchmarks": [ | |
| "laya--ag-news-choice-test-v1", | |
| "arc-choice-test-v1", | |
| "arct-choice-test-v1", | |
| "babi-nli-noul-test-v1", | |
| "banking77-choice-test-v1", | |
| "civil-comments-noul-test-v1", | |
| "clinc-choice-test-v1", | |
| "contract-nli-choice-test-v1", | |
| "corr2cause-choice-test-v1", | |
| "creak-noul-test-v1", | |
| "dbpedia-choice-test-v1", | |
| "defeasible-nli-choice-test-v1", | |
| "ethics-choice-test-v1", | |
| "ethics-noul-test-v1", | |
| "ethos-noul-test-v1", | |
| "fol-nli-choice-test-v1", | |
| "go-emotions-noul-test-v1", | |
| "hans-noul-test-v1", | |
| "hh-rlhf-choice-test-v1", | |
| "logical-entailment-noul-test-v1", | |
| "lonli-choice-test-v1", | |
| "massive-choice-test-v1", | |
| "mtop-choice-test-v1", | |
| "openbookqa-choice-test-v1", | |
| "patent-similarity-score-test-v1", | |
| "paws-noul-test-v1", | |
| "poem-sentiment-choice-test-v1", | |
| "qasper-noul-test-v1", | |
| "quartz-choice-test-v1", | |
| "ruletaker-noul-test-v1", | |
| "scicite-choice-test-v1", | |
| "scitail-noul-test-v1", | |
| "scone-choice-test-v1", | |
| "sdoh-nli-noul-test-v1", | |
| "snli-choice-test-v1", | |
| "spartqa-choice-test-v1", | |
| "stepgame-choice-test-v1", | |
| "temporal-nli-choice-test-v1", | |
| "tracie-noul-test-v1", | |
| "winowhy-noul-test-v1", | |
| "wiqa-choice-test-v1" | |
| ] | |
| }, | |
| "hypotheses": { | |
| "H1": { | |
| "test": "V2 ablations (same 12K questions): votes-mixed readout minus gold-only readout, in-distribution accuracy (procedural + tasksource, n=1125), pts", | |
| "point": 1.5111111111111075, | |
| "ci95": [ | |
| -0.4444444444444362, | |
| 3.3777777777777795 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.0651, | |
| "n_boot": 10000, | |
| "all_eval": { | |
| "point": -1.1944138184490916, | |
| "ci95": [ | |
| -2.131569276001477, | |
| -0.2756339581036382 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.9951, | |
| "n_boot": 10000 | |
| }, | |
| "rule": "CI lower bound > 0", | |
| "confirmed": false | |
| }, | |
| "H2": { | |
| "test": "disagreement AUROC (readout 1 - max p vs its own reasoner's answer, 1,498), rl60 minus base and minus v1", | |
| "by_comparison": { | |
| "rl60-base": { | |
| "point": 0.059961135720413394, | |
| "ci95": [ | |
| 0.03013448403970405, | |
| 0.08972870881776687 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000, | |
| "ok": true, | |
| "ci_lower_ge_0.03": true | |
| }, | |
| "rl60-v1": { | |
| "point": 0.05315260777766628, | |
| "ci95": [ | |
| 0.026016003868889737, | |
| 0.0800455701108224 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0002, | |
| "n_boot": 10000, | |
| "ok": true, | |
| "ci_lower_ge_0.03": false | |
| }, | |
| "v2-base": { | |
| "point": 0.06588399082875795, | |
| "ci95": [ | |
| 0.03548144843090986, | |
| 0.09598063336036833 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000, | |
| "ok": true, | |
| "ci_lower_ge_0.03": true | |
| } | |
| }, | |
| "levels": { | |
| "base": 0.7728567119155354, | |
| "v1": 0.7796652398582825, | |
| "v2": 0.8387407027442934, | |
| "rl60": 0.8328178476359488, | |
| "rl40": 0.8391960206975687 | |
| }, | |
| "rule": "delta >= +0.03 and CI lower bound > 0 for both rl60 comparisons (metrics.py rule)", | |
| "confirmed": true | |
| }, | |
| "H3": { | |
| "test": "trained reasoner (rl60) minus base reasoner, thinking accuracy on 1,498, pts", | |
| "point": -0.9345794392523366, | |
| "ci95": [ | |
| -2.603471295060078, | |
| 0.7343124165554049 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.8674, | |
| "n_boot": 10000, | |
| "v2_minus_base": { | |
| "point": -2.87049399198932, | |
| "ci95": [ | |
| -4.672897196261683, | |
| -1.001335113484647 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.9987, | |
| "n_boot": 10000 | |
| }, | |
| "rl40_minus_base": { | |
| "point": -0.4672897196261627, | |
| "ci95": [ | |
| -2.202937249666226, | |
| 1.3351134846461887 | |
| ], | |
| "excludes_0": false, | |
| "boot_share_le_0": 0.7073, | |
| "n_boot": 10000 | |
| }, | |
| "rule": "point >= -1.0 pts (metrics.py rule); stricter non-inferiority: CI lower bound >= -1.0", | |
| "confirmed": true, | |
| "non_inferiority_shown": false, | |
| "v2_confirmed": false | |
| }, | |
| "H4": { | |
| "test": "rl60 gap recovered at f=0.3 (lowest-margin routing, 1,498)", | |
| "point": 0.7946428571428577, | |
| "ci95": [ | |
| 0.6399999999999991, | |
| 0.9726180283757333 | |
| ], | |
| "excludes_0": true, | |
| "boot_share_le_0": 0.0, | |
| "n_boot": 10000, | |
| "rule": "point >= 0.80 (metrics.py rule)", | |
| "confirmed": false, | |
| "ci_contains_0.80": true | |
| } | |
| } | |
| } |
Xet Storage Details
- Size:
- 41.3 kB
- Xet hash:
- 91948ba985638f68fe534b339740bed51961653160789a8bc60395a2506340af
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.