Buckets:
| { | |
| "paper": { | |
| "title": "Calibrated Preference Learning: The Case of Label Ranking", | |
| "orid": "STcIzNrUBB", | |
| "arxiv": "2605.30447" | |
| }, | |
| "claims": [ | |
| "Theorem 4.2 proves that full-rank calibration implies both sub-k calibration and top-k calibration, but Table 1 gives a counterexample showing sub-k calibration does not imply full-rank calibration (Section 4, Theorem 4.2, Table 1).", | |
| "Table 2 provides a counterexample showing top-k calibration does not imply full-rank calibration, and that sub-k and top-k calibration are mutually incomparable notions (Section 4, Table 2).", | |
| "Theorem 4.3 shows sub-k calibration implies rankwise sub-k calibration and top-k calibration implies rankwise top-k calibration, establishing a strict hierarchy of calibration notions defined in Definitions 1-7 (Section 4, Theorem 4.3).", | |
| "On RewardBench2, Plackett-Luce and Mallows-Model reward models are found to be generally poorly calibrated (measured via top-1 Expected Calibration Error), while RPC achieves the strongest calibration for pairwise rankings (Section 5.2).", | |
| "Top-1 ECE correlates strongly with reward-model accuracy in the Focus and Safety categories of RewardBench2 but only weakly in the Math and Precise-IF categories, showing calibration is an imperfect proxy for benchmark accuracy (Section 5.2)." | |
| ], | |
| "source_audit": { | |
| "paper_tex_sha256": "fde5ec1b099be986a9de4e407525381a86f26f5217564bf365afcddd43df9475", | |
| "theorem_4_2_present": true, | |
| "theorem_4_3_present": true, | |
| "table_1_present": true, | |
| "table_2_present": true, | |
| "table_3_present": true, | |
| "official_code_commit": "dc0b53324cfc98625517f6e7cdeb5c65a77b89a6", | |
| "official_code_url": "https://github.com/Advueu963/Calibrated_Preference_Learning/tree/dc0b53324cfc98625517f6e7cdeb5c65a77b89a6" | |
| }, | |
| "claim_1": { | |
| "table1": { | |
| "sub2_max_error": "0", | |
| "top1_max_error": "1/6", | |
| "full_rank_l1_error": "2/3", | |
| "sub2_calibrated_not_full_rank": true | |
| }, | |
| "full_rank_random_checks": { | |
| "count": 200, | |
| "max_error": 0.0 | |
| } | |
| }, | |
| "claim_2": { | |
| "table3": { | |
| "top1_max_error": "0", | |
| "sub2_max_error": "1/6", | |
| "full_rank_l1_error": "1", | |
| "top1_calibrated_not_full_rank": true | |
| }, | |
| "table2": { | |
| "max_sub2_violation": "1/6", | |
| "max_top1_violation": "1/6", | |
| "rankwise_but_neither_sub2_nor_top1": true | |
| }, | |
| "mutual_incomparability": true, | |
| "numbering_note": "The current source's Table 2 is the rankwise-but-neither counterexample; the top-1-not-full-rank counterexample is Table 3. The registered claim's substance is true, but its table anchor is inaccurate." | |
| }, | |
| "claim_3": { | |
| "coordinate_implication": true, | |
| "strictness_witnessed_by_tables": true | |
| }, | |
| "claim_4": { | |
| "label_ranking": { | |
| "datasets": [ | |
| { | |
| "dataset": "political", | |
| "csv": "official-code/results/political/subk_ece_results_political_1.0_95_prob_mass_abs_linear.csv", | |
| "rpc": 0.11606087821488724, | |
| "plackett_luce": 0.3353780261063616, | |
| "mallows": 0.41621835440484567, | |
| "rpc_best_of_three": true | |
| }, | |
| { | |
| "dataset": "movies", | |
| "csv": "official-code/results/movies/subk_ece_results_movies_0.0_95_prob_mass_abs_linear.csv", | |
| "rpc": 0.07066460016122396, | |
| "plackett_luce": 0.5004784664720409, | |
| "mallows": 0.5004784624255281, | |
| "rpc_best_of_three": true | |
| }, | |
| { | |
| "dataset": "glass", | |
| "csv": "official-code/results/glass/subk_ece_results_glass_1.0_95_prob_mass_abs_linear.csv", | |
| "rpc": 0.0887899778194392, | |
| "plackett_luce": 0.22729745958076847, | |
| "mallows": 0.2775441297082739, | |
| "rpc_best_of_three": true | |
| } | |
| ], | |
| "rpc_best_count": 3, | |
| "official_commit": "dc0b53324cfc98625517f6e7cdeb5c65a77b89a6" | |
| }, | |
| "rewardbench2": { | |
| "dataset_url": "https://huggingface.co/datasets/allenai/reward-bench-2-results", | |
| "dataset_revision": "b19c7033e964187d12e74a43a07f2d727a3d37e5", | |
| "models_with_scores": 60, | |
| "best_model": { | |
| "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", | |
| "ece": 0.13476999174795454, | |
| "n_examples": 1865, | |
| "score": 84.12779216039524, | |
| "factuality": 84.63157894736842, | |
| "precise_if": 66.25, | |
| "math": 77.59562841530054, | |
| "safety": 96.66666666666669, | |
| "focus": 98.38383838383838 | |
| }, | |
| "best_10_mean_ece": 0.22393760010742372, | |
| "median_ece": 0.46872034234741833 | |
| }, | |
| "scope_note": "The challenge wording conflates Section 5.1's PL/Mallows/RPC label-ranking comparison with Section 5.2's Bradley-Terry RewardBench2 study. Both components are audited separately." | |
| }, | |
| "claim_5": { | |
| "correlations": { | |
| "score": { | |
| "kendall_tau": 0.6915254237288135, | |
| "p_value": 5.875884895968332e-15 | |
| }, | |
| "factuality": { | |
| "kendall_tau": 0.5882356704893549, | |
| "p_value": 3.2743073277223836e-11 | |
| }, | |
| "precise_if": { | |
| "kendall_tau": 0.5312423782820873, | |
| "p_value": 3.08971978021332e-09 | |
| }, | |
| "math": { | |
| "kendall_tau": 0.4956061106975527, | |
| "p_value": 2.54616399156624e-08 | |
| }, | |
| "safety": { | |
| "kendall_tau": 0.6749734188228499, | |
| "p_value": 2.868043910010452e-14 | |
| }, | |
| "focus": { | |
| "kendall_tau": 0.698359806738388, | |
| "p_value": 3.5090802097808933e-15 | |
| } | |
| }, | |
| "destructive_permutation_control": { | |
| "precise_if": { | |
| "mean_tau": -0.003936231242529776, | |
| "q025": -0.17806351483258973, | |
| "q975": 0.16775924456418712 | |
| }, | |
| "math": { | |
| "mean_tau": -0.001382928391362243, | |
| "q025": -0.16466271753473674, | |
| "q975": 0.16926112474739438 | |
| }, | |
| "safety": { | |
| "mean_tau": -0.0011393007706976283, | |
| "q025": -0.17726546037121912, | |
| "q975": 0.17398119373600712 | |
| }, | |
| "focus": { | |
| "mean_tau": -0.0029179442168096666, | |
| "q025": -0.17102458152053554, | |
| "q975": 0.1703737599826513 | |
| } | |
| } | |
| }, | |
| "rewardbench_rows": [ | |
| { | |
| "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", | |
| "ece": 0.13476999174795454, | |
| "n_examples": 1865, | |
| "score": 84.12779216039524, | |
| "factuality": 84.63157894736842, | |
| "precise_if": 66.25, | |
| "math": 77.59562841530054, | |
| "safety": 96.66666666666669, | |
| "focus": 98.38383838383838 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-V2-Qwen3-8B", | |
| "ece": 0.19052674352425994, | |
| "n_examples": 1865, | |
| "score": 78.37409729353149, | |
| "factuality": 79.89473684210526, | |
| "precise_if": 50.0, | |
| "math": 77.04918032786885, | |
| "safety": 94.0, | |
| "focus": 96.36363636363636 | |
| }, | |
| { | |
| "model": "infly/INF-ORM-Llama3.1-70B", | |
| "ece": 0.23938761733076275, | |
| "n_examples": 1865, | |
| "score": 76.48295733317504, | |
| "factuality": 74.10526315789474, | |
| "precise_if": 41.875, | |
| "math": 69.94535519125684, | |
| "safety": 96.44444444444444, | |
| "focus": 90.30303030303033 | |
| }, | |
| { | |
| "model": "anthropic/claude-opus-4-20250514", | |
| "ece": 0.44304509051326607, | |
| "n_examples": 102, | |
| "score": 76.48277145949112, | |
| "factuality": 82.66666666666666, | |
| "precise_if": 41.875, | |
| "math": 74.90892531876139, | |
| "safety": 89.53703703703702, | |
| "focus": 86.16161616161615 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-70B-Instruct-RM-RB2", | |
| "ece": 0.3454970143401523, | |
| "n_examples": 1865, | |
| "score": 76.05646300951571, | |
| "factuality": 81.26315789473684, | |
| "precise_if": 41.875, | |
| "math": 69.94535519125684, | |
| "safety": 88.44444444444444, | |
| "focus": 86.46464646464646 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-V2-Qwen3-4B", | |
| "ece": 0.21117089642114426, | |
| "n_examples": 1865, | |
| "score": 75.51022543280415, | |
| "factuality": 77.36842105263158, | |
| "precise_if": 46.25, | |
| "math": 73.224043715847, | |
| "safety": 92.22222222222224, | |
| "focus": 96.56565656565655 | |
| }, | |
| { | |
| "model": "anthropic/claude-3-7-sonnet-20250219", | |
| "ece": 0.47587195310274616, | |
| "n_examples": 102, | |
| "score": 75.38767274570088, | |
| "factuality": 73.26315789473684, | |
| "precise_if": 54.37499999999999, | |
| "math": 75.0, | |
| "safety": 90.33333333333331, | |
| "focus": 92.12121212121212 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-V2-Llama-3.2-3B", | |
| "ece": 0.20632375848843673, | |
| "n_examples": 1865, | |
| "score": 74.66480123197486, | |
| "factuality": 76.21052631578948, | |
| "precise_if": 45.625, | |
| "math": 69.39890710382514, | |
| "safety": 93.11111111111111, | |
| "focus": 95.95959595959596 | |
| }, | |
| { | |
| "model": "LxzGordon/URM-LLaMa-3.1-8B", | |
| "ece": 0.2666059664445265, | |
| "n_examples": 1865, | |
| "score": 73.94354072783193, | |
| "factuality": 68.84210526315789, | |
| "precise_if": 45.0, | |
| "math": 63.934426229508205, | |
| "safety": 91.7777777777778, | |
| "focus": 97.57575757575758 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-Llama-3.1-8B", | |
| "ece": 0.21604154406292006, | |
| "n_examples": 1865, | |
| "score": 73.1387048400328, | |
| "factuality": 69.89473684210526, | |
| "precise_if": 42.5, | |
| "math": 62.841530054644814, | |
| "safety": 93.33333333333331, | |
| "focus": 96.16161616161617 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-8B-Instruct-RM-RB2", | |
| "ece": 0.3829611922762887, | |
| "n_examples": 1865, | |
| "score": 72.84710237854704, | |
| "factuality": 74.31578947368422, | |
| "precise_if": 44.375, | |
| "math": 61.74863387978142, | |
| "safety": 89.55555555555556, | |
| "focus": 90.7070707070707 | |
| }, | |
| { | |
| "model": "ShikaiChen/LDL-Reward-Gemma-2-27B-v0.1", | |
| "ece": 0.5628945129050107, | |
| "n_examples": 1865, | |
| "score": 72.48764297589858, | |
| "factuality": 75.57894736842105, | |
| "precise_if": 35.0, | |
| "math": 64.48087431693989, | |
| "safety": 92.22222222222224, | |
| "focus": 91.31313131313132 | |
| }, | |
| { | |
| "model": "openai/gpt-4.1-2025-04-14", | |
| "ece": 0.429620770444169, | |
| "n_examples": 102, | |
| "score": 72.31731857413882, | |
| "factuality": 82.89473684210526, | |
| "precise_if": 39.73958333333334, | |
| "math": 65.20947176684881, | |
| "safety": 87.25925925925924, | |
| "focus": 73.38383838383838 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-Tulu-3-70B-SFT-RM-RB2", | |
| "ece": 0.3805613091161596, | |
| "n_examples": 1865, | |
| "score": 72.20387212509466, | |
| "factuality": 80.84210526315789, | |
| "precise_if": 36.875, | |
| "math": 67.75956284153006, | |
| "safety": 86.8888888888889, | |
| "focus": 77.77777777777779 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", | |
| "ece": 0.2248423768646985, | |
| "n_examples": 1865, | |
| "score": 71.7452543316723, | |
| "factuality": 69.6842105263158, | |
| "precise_if": 40.625, | |
| "math": 60.10928961748634, | |
| "safety": 94.2222222222222, | |
| "focus": 94.14141414141412 | |
| }, | |
| { | |
| "model": "anthropic/claude-sonnet-4-20250514", | |
| "ece": 0.4666141658421129, | |
| "n_examples": 102, | |
| "score": 71.16513658188997, | |
| "factuality": 76.12280701754385, | |
| "precise_if": 35.937500000000014, | |
| "math": 70.49180327868852, | |
| "safety": 89.09259259259258, | |
| "focus": 75.95959595959594 | |
| }, | |
| { | |
| "model": "HFXM/RAMO-Llama3.1-8B", | |
| "ece": 0.33136097555912775, | |
| "n_examples": 1865, | |
| "score": 69.17258753860413, | |
| "factuality": 65.47368421052632, | |
| "precise_if": 37.5, | |
| "math": 56.28415300546448, | |
| "safety": 97.55555555555556, | |
| "focus": 90.7070707070707 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-VL-Reward-7B", | |
| "ece": 0.26843480120480384, | |
| "n_examples": 1865, | |
| "score": 68.84669257606988, | |
| "factuality": 60.631578947368425, | |
| "precise_if": 35.0, | |
| "math": 63.387978142076506, | |
| "safety": 89.11111111111111, | |
| "focus": 89.0909090909091 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-Tulu-3-8B-RL-RM-RB2", | |
| "ece": 0.35348883907419754, | |
| "n_examples": 1865, | |
| "score": 68.71219205900773, | |
| "factuality": 76.42105263157895, | |
| "precise_if": 40.0, | |
| "math": 61.74863387978142, | |
| "safety": 86.44444444444444, | |
| "focus": 84.84848484848484 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-Tulu-3-8B-DPO-RM-RB2", | |
| "ece": 0.3519554515743213, | |
| "n_examples": 1865, | |
| "score": 68.69585657154352, | |
| "factuality": 75.1578947368421, | |
| "precise_if": 38.75, | |
| "math": 62.841530054644814, | |
| "safety": 86.0, | |
| "focus": 85.45454545454545 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-Tulu-3-8B-SFT-RM-RB2", | |
| "ece": 0.28127230498472994, | |
| "n_examples": 1865, | |
| "score": 68.20514558353437, | |
| "factuality": 73.26315789473684, | |
| "precise_if": 38.75, | |
| "math": 57.92349726775956, | |
| "safety": 89.77777777777777, | |
| "focus": 88.88888888888889 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-V2-Qwen3-1.7B", | |
| "ece": 0.2957017764976967, | |
| "n_examples": 1865, | |
| "score": 68.17585285009079, | |
| "factuality": 65.6842105263158, | |
| "precise_if": 44.375, | |
| "math": 72.6775956284153, | |
| "safety": 89.11111111111111, | |
| "focus": 88.48484848484848 | |
| }, | |
| { | |
| "model": "Ray2333/GRM-Llama3-8B-rewardmodel-ft", | |
| "ece": 0.293474992842568, | |
| "n_examples": 1865, | |
| "score": 67.66019726388538, | |
| "factuality": 62.73684210526316, | |
| "precise_if": 35.0, | |
| "math": 58.46994535519126, | |
| "safety": 92.22222222222224, | |
| "focus": 89.29292929292929 | |
| }, | |
| { | |
| "model": "openai/gpt-4.1-mini-2025-04-14", | |
| "ece": 0.47588383034528187, | |
| "n_examples": 102, | |
| "score": 65.7349500114784, | |
| "factuality": 60.8421052631579, | |
| "precise_if": 41.25, | |
| "math": 72.1311475409836, | |
| "safety": 72.64814814814811, | |
| "focus": 73.53535353535354 | |
| }, | |
| { | |
| "model": "openai/gpt-4o-2024-08-06", | |
| "ece": 0.41589317817039906, | |
| "n_examples": 102, | |
| "score": 64.92828668947554, | |
| "factuality": 56.84210526315789, | |
| "precise_if": 33.125, | |
| "math": 62.29508196721312, | |
| "safety": 86.1851851851852, | |
| "focus": 72.92929292929293 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-8B-Base-RM-RB2", | |
| "ece": 0.3291385082177684, | |
| "n_examples": 1865, | |
| "score": 64.90187760825604, | |
| "factuality": 72.0, | |
| "precise_if": 36.25, | |
| "math": 61.20218579234973, | |
| "safety": 82.66666666666667, | |
| "focus": 83.23232323232324 | |
| }, | |
| { | |
| "model": "NCSOFT/Llama-3-OffsetBias-RM-8B", | |
| "ece": 0.43174056192181015, | |
| "n_examples": 1865, | |
| "score": 64.79939430060925, | |
| "factuality": 60.8421052631579, | |
| "precise_if": 40.0, | |
| "math": 51.91256830601093, | |
| "safety": 72.22222222222221, | |
| "focus": 95.95959595959596 | |
| }, | |
| { | |
| "model": "anthropic/claude-3-5-sonnet-20240620", | |
| "ece": 0.4708265188527238, | |
| "n_examples": 102, | |
| "score": 64.66244541750068, | |
| "factuality": 52.84210526315789, | |
| "precise_if": 38.75, | |
| "math": 56.830601092896174, | |
| "safety": 85.18518518518516, | |
| "focus": 86.96969696969697 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-V2-Llama-3.2-1B", | |
| "ece": 0.3252607479986218, | |
| "n_examples": 1865, | |
| "score": 64.37645875381985, | |
| "factuality": 60.8421052631579, | |
| "precise_if": 45.625, | |
| "math": 60.10928961748634, | |
| "safety": 87.33333333333333, | |
| "focus": 89.29292929292929 | |
| }, | |
| { | |
| "model": "Qwen/WorldPM-72B", | |
| "ece": 0.5629368528218481, | |
| "n_examples": 1865, | |
| "score": 63.3269114330196, | |
| "factuality": 70.73684210526315, | |
| "precise_if": 31.25, | |
| "math": 65.57377049180327, | |
| "safety": 85.33333333333334, | |
| "focus": 91.71717171717172 | |
| }, | |
| { | |
| "model": "sfairXC/FsfairX-LLaMA3-RM-v0.1", | |
| "ece": 0.4515965595882692, | |
| "n_examples": 1865, | |
| "score": 62.91859600227345, | |
| "factuality": 59.15789473684211, | |
| "precise_if": 41.875, | |
| "math": 62.841530054644814, | |
| "safety": 76.66666666666667, | |
| "focus": 70.5050505050505 | |
| }, | |
| { | |
| "model": "Skywork/Skywork-Reward-V2-Qwen3-0.6B", | |
| "ece": 0.39085231027634043, | |
| "n_examples": 1865, | |
| "score": 61.24950037720884, | |
| "factuality": 57.99999999999999, | |
| "precise_if": 40.0, | |
| "math": 71.58469945355192, | |
| "safety": 84.44444444444444, | |
| "focus": 79.4949494949495 | |
| }, | |
| { | |
| "model": "Ray2333/GRM-llama3-8B-sftreg", | |
| "ece": 0.41225477535946514, | |
| "n_examples": 1865, | |
| "score": 60.88793933636373, | |
| "factuality": 61.89473684210526, | |
| "precise_if": 38.75, | |
| "math": 57.92349726775956, | |
| "safety": 78.66666666666666, | |
| "focus": 68.28282828282828 | |
| }, | |
| { | |
| "model": "Ray2333/GRM-gemma2-2B-rewardmodel-ft", | |
| "ece": 0.36672628866425133, | |
| "n_examples": 1865, | |
| "score": 59.66119140947769, | |
| "factuality": 53.05263157894737, | |
| "precise_if": 31.25, | |
| "math": 59.01639344262295, | |
| "safety": 92.22222222222224, | |
| "focus": 74.54545454545455 | |
| }, | |
| { | |
| "model": "weqweasdas/RM-Mistral-7B", | |
| "ece": 0.47281749801857487, | |
| "n_examples": 1865, | |
| "score": 59.60117398651728, | |
| "factuality": 59.36842105263158, | |
| "precise_if": 34.375, | |
| "math": 59.56284153005464, | |
| "safety": 69.11111111111111, | |
| "focus": 72.92929292929293 | |
| }, | |
| { | |
| "model": "allenai/Llama-3.1-Tulu-3-8B-RM", | |
| "ece": 0.5418628632379631, | |
| "n_examples": 1865, | |
| "score": 58.99751678794565, | |
| "factuality": 74.52631578947368, | |
| "precise_if": 34.6875, | |
| "math": 64.48087431693989, | |
| "safety": 74.22222222222223, | |
| "focus": 53.63636363636364 | |
| }, | |
| { | |
| "model": "Ray2333/GRM-llama3-8B-distill", | |
| "ece": 0.5049069764842984, | |
| "n_examples": 1865, | |
| "score": 58.903971632622586, | |
| "factuality": 58.73684210526315, | |
| "precise_if": 38.75, | |
| "math": 59.01639344262295, | |
| "safety": 72.22222222222221, | |
| "focus": 67.27272727272727 | |
| }, | |
| { | |
| "model": "hendrydong/Mistral-RM-for-RAFT-GSHF-v0", | |
| "ece": 0.4852071041963555, | |
| "n_examples": 1865, | |
| "score": 58.51045929269105, | |
| "factuality": 57.78947368421052, | |
| "precise_if": 36.25, | |
| "math": 60.10928961748634, | |
| "safety": 69.55555555555556, | |
| "focus": 67.47474747474747 | |
| }, | |
| { | |
| "model": "openbmb/Eurus-RM-7b", | |
| "ece": 0.38244755652313694, | |
| "n_examples": 1865, | |
| "score": 58.05660762038656, | |
| "factuality": 60.0, | |
| "precise_if": 34.375, | |
| "math": 56.830601092896174, | |
| "safety": 62.66666666666667, | |
| "focus": 74.74747474747475 | |
| }, | |
| { | |
| "model": "openai/gpt-4o-mini-2024-07-18", | |
| "ece": 0.48926280757784807, | |
| "n_examples": 102, | |
| "score": 57.96141949652715, | |
| "factuality": 41.05263157894737, | |
| "precise_if": 34.375, | |
| "math": 51.91256830601093, | |
| "safety": 76.66666666666664, | |
| "focus": 74.14141414141415 | |
| }, | |
| { | |
| "model": "anthropic/claude-3-opus-20240229", | |
| "ece": 0.5237197690555281, | |
| "n_examples": 102, | |
| "score": 57.44034084018399, | |
| "factuality": 53.89473684210526, | |
| "precise_if": 33.125, | |
| "math": 51.36612021857923, | |
| "safety": 83.77777777777777, | |
| "focus": 66.46464646464646 | |
| }, | |
| { | |
| "model": "CIR-AMS/BTRM_Qwen2_7b_0613", | |
| "ece": 0.5160768592774613, | |
| "n_examples": 1865, | |
| "score": 57.36341394412572, | |
| "factuality": 53.47368421052632, | |
| "precise_if": 35.625, | |
| "math": 60.65573770491803, | |
| "safety": 71.77777777777777, | |
| "focus": 57.37373737373738 | |
| }, | |
| { | |
| "model": "internlm/internlm2-20b-reward", | |
| "ece": 0.6009934794349481, | |
| "n_examples": 1865, | |
| "score": 56.27800895371716, | |
| "factuality": 55.57894736842105, | |
| "precise_if": 36.25, | |
| "math": 57.377049180327866, | |
| "safety": 61.11111111111112, | |
| "focus": 72.52525252525253 | |
| }, | |
| { | |
| "model": "internlm/internlm2-7b-reward", | |
| "ece": 0.6132021913976878, | |
| "n_examples": 1865, | |
| "score": 53.34773837781889, | |
| "factuality": 42.10526315789473, | |
| "precise_if": 40.0, | |
| "math": 56.28415300546448, | |
| "safety": 59.55555555555555, | |
| "focus": 70.5050505050505 | |
| }, | |
| { | |
| "model": "google/gemini-1.5-flash-8b", | |
| "ece": 0.8061676283717575, | |
| "n_examples": 102, | |
| "score": 48.51363189431879, | |
| "factuality": 46.10526315789474, | |
| "precise_if": 36.25, | |
| "math": 50.81967213114754, | |
| "safety": 66.22222222222223, | |
| "focus": 67.47474747474747 | |
| }, | |
| { | |
| "model": "openai/gpt-4.1-nano-2025-04-14", | |
| "ece": 0.5344009637193005, | |
| "n_examples": 102, | |
| "score": 48.49148497149796, | |
| "factuality": 46.456140350877234, | |
| "precise_if": 25.78125, | |
| "math": 50.40983606557376, | |
| "safety": 71.55555555555553, | |
| "focus": 46.59932659932663 | |
| }, | |
| { | |
| "model": "weqweasdas/RM-Gemma-7B", | |
| "ece": 0.6004406343539396, | |
| "n_examples": 1865, | |
| "score": 48.25502869477731, | |
| "factuality": 49.26315789473684, | |
| "precise_if": 39.375, | |
| "math": 60.65573770491803, | |
| "safety": 48.22222222222222, | |
| "focus": 49.696969696969695 | |
| }, | |
| { | |
| "model": "openbmb/UltraRM-13b", | |
| "ece": 0.5513903214584469, | |
| "n_examples": 1865, | |
| "score": 46.83473363506545, | |
| "factuality": 50.63157894736842, | |
| "precise_if": 33.125, | |
| "math": 55.19125683060109, | |
| "safety": 50.888888888888886, | |
| "focus": 60.80808080808081 | |
| }, | |
| { | |
| "model": "Nexusflow/Starling-RM-34B", | |
| "ece": 0.6136179212108072, | |
| "n_examples": 1865, | |
| "score": 45.53255691061264, | |
| "factuality": 45.89473684210526, | |
| "precise_if": 31.875, | |
| "math": 61.74863387978142, | |
| "safety": 75.55555555555556, | |
| "focus": 48.08080808080808 | |
| }, | |
| { | |
| "model": "internlm/internlm2-1_8b-reward", | |
| "ece": 0.6727187813665824, | |
| "n_examples": 1865, | |
| "score": 39.02376339707181, | |
| "factuality": 27.57894736842105, | |
| "precise_if": 36.25, | |
| "math": 44.26229508196721, | |
| "safety": 47.11111111111111, | |
| "focus": 59.59595959595959 | |
| }, | |
| { | |
| "model": "anthropic/claude-3-haiku-20240307", | |
| "ece": 0.8125813921703674, | |
| "n_examples": 102, | |
| "score": 37.11004606544254, | |
| "factuality": 40.42105263157895, | |
| "precise_if": 28.125, | |
| "math": 35.51912568306011, | |
| "safety": 59.5, | |
| "focus": 50.1010101010101 | |
| }, | |
| { | |
| "model": "PKU-Alignment/beaver-7b-v1.0-cost", | |
| "ece": 0.6213311133259143, | |
| "n_examples": 1865, | |
| "score": 33.32169419042497, | |
| "factuality": 32.631578947368425, | |
| "precise_if": 23.125, | |
| "math": 39.89071038251366, | |
| "safety": 75.88888888888889, | |
| "focus": 29.393939393939394 | |
| }, | |
| { | |
| "model": "PKU-Alignment/beaver-7b-v2.0-cost", | |
| "ece": 0.6505662098496096, | |
| "n_examples": 1865, | |
| "score": 33.26126668695694, | |
| "factuality": 37.89473684210527, | |
| "precise_if": 27.500000000000004, | |
| "math": 33.33333333333333, | |
| "safety": 73.55555555555556, | |
| "focus": 28.28282828282828 | |
| }, | |
| { | |
| "model": "OpenAssistant/reward-model-deberta-v3-large-v2", | |
| "ece": 0.6876232170863141, | |
| "n_examples": 1865, | |
| "score": 32.00225906524769, | |
| "factuality": 38.526315789473685, | |
| "precise_if": 26.875, | |
| "math": 50.27322404371585, | |
| "safety": 36.66666666666666, | |
| "focus": 27.67676767676768 | |
| }, | |
| { | |
| "model": "weqweasdas/RM-Gemma-2B", | |
| "ece": 0.7263672822485251, | |
| "n_examples": 1865, | |
| "score": 30.566279281365752, | |
| "factuality": 37.05263157894737, | |
| "precise_if": 28.125, | |
| "math": 43.16939890710383, | |
| "safety": 33.111111111111114, | |
| "focus": 23.43434343434344 | |
| }, | |
| { | |
| "model": "OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1", | |
| "ece": 0.7593884215827624, | |
| "n_examples": 1865, | |
| "score": 26.53180836792417, | |
| "factuality": 39.78947368421053, | |
| "precise_if": 28.75, | |
| "math": 37.704918032786885, | |
| "safety": 32.88888888888889, | |
| "focus": 15.353535353535351 | |
| }, | |
| { | |
| "model": "OpenAssistant/oasst-rm-2.1-pythia-1.4b-epoch-2.5", | |
| "ece": 0.7202181761332691, | |
| "n_examples": 1865, | |
| "score": 26.47938178923201, | |
| "factuality": 31.789473684210527, | |
| "precise_if": 26.25, | |
| "math": 39.34426229508197, | |
| "safety": 32.44444444444444, | |
| "focus": 27.070707070707076 | |
| }, | |
| { | |
| "model": "PKU-Alignment/beaver-7b-v2.0-reward", | |
| "ece": 0.756975347321237, | |
| "n_examples": 1865, | |
| "score": 25.435434978750283, | |
| "factuality": 21.684210526315788, | |
| "precise_if": 25.625, | |
| "math": 38.25136612021858, | |
| "safety": 31.555555555555557, | |
| "focus": 26.060606060606062 | |
| }, | |
| { | |
| "model": "weqweasdas/hh_rlhf_rm_open_llama_3b", | |
| "ece": 0.7492188305498545, | |
| "n_examples": 1865, | |
| "score": 24.98045597204856, | |
| "factuality": 36.42105263157895, | |
| "precise_if": 27.500000000000004, | |
| "math": 34.97267759562842, | |
| "safety": 24.0, | |
| "focus": 23.83838383838384 | |
| }, | |
| { | |
| "model": "PKU-Alignment/beaver-7b-v1.0-reward", | |
| "ece": 0.819336116115328, | |
| "n_examples": 1865, | |
| "score": 16.05736504908619, | |
| "factuality": 21.052631578947366, | |
| "precise_if": 29.375, | |
| "math": 26.229508196721312, | |
| "safety": 14.22222222222222, | |
| "focus": 6.4646464646464645 | |
| } | |
| ], | |
| "verdicts": { | |
| "claim_1": "supported_exact_fraction_and_random_numerical_audit", | |
| "claim_2": "supported_with_table_numbering_correction", | |
| "claim_3": "supported_numerical_implication_audit_not_proof", | |
| "claim_4": "supported_with_section_scope_clarification", | |
| "claim_5": "supported_public_data_recomputation" | |
| }, | |
| "scope": "Exact-fraction theory audit; official immutable CSV aggregation; full available RewardBench2 score recomputation. No model training.", | |
| "wall_time_seconds": 32.77384108499973 | |
| } |
Xet Storage Details
- Size:
- 26.1 kB
- Xet hash:
- 8fbe2d9b8b96a56eaef3d214e2a7efd7f7cec16d5b9fd3ec4847a977f1cf450b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.