| { |
| "provenance": "Re-run of the authors' released scripts (github.com/hlzhang109/prescriptive-scaling) on the authors' released tables, on this machine.", |
| "table1_authors_code": { |
| "BBH Raw": 0.6995598819257327, |
| "GPQA Raw": 0.4244877017896367, |
| "IFEval Raw": 0.827845230576459, |
| "MATH Lvl 5 Raw": 0.5395001305416097, |
| "MMLU-PRO Raw": 0.5626259737685013, |
| "MUSR Raw": 0.535245164843684 |
| }, |
| "table2_authors_code": { |
| "Constant": { |
| "pinball_ID": 0.005352683799141467, |
| "pinball_OOD": 0.006233912544681612, |
| "calib_ID": 0.04123274010277254, |
| "calib_OOD": 0.03597267501958549 |
| }, |
| "Binwise": { |
| "pinball_ID": 0.004014274443578322, |
| "pinball_OOD": 0.004995430505610828, |
| "calib_ID": 0.016646429776157354, |
| "calib_OOD": 0.028066352913576547 |
| }, |
| "I-spline": { |
| "pinball_ID": 0.004003277885706522, |
| "pinball_OOD": 0.004918276548778966, |
| "calib_ID": 0.018259079529548153, |
| "calib_OOD": 0.024059338259677387 |
| }, |
| "Sigmoid": { |
| "pinball_ID": 0.004082793246777277, |
| "pinball_OOD": 0.00492990898200165, |
| "calib_ID": 0.018373223589866268, |
| "calib_OOD": 0.02205737092377908 |
| } |
| }, |
| "n_task_period_pairs": 18, |
| "claim4_authors_code": { |
| "BBH Raw": { |
| "k1_MAE_OOS_macro": 0.0162621, |
| "k2_MAE_OOS_macro": 0.0157885, |
| "k3_MAE_OOS_macro": 0.0109922, |
| "k1_signed_coverage_micro": 0.013671086497890289, |
| "k2_signed_coverage_micro": 0.01554063991081382, |
| "k3_signed_coverage_micro": 0.007951856224899597, |
| "mean_MAE_OOS_macro": 0.0143476, |
| "mean_signed_coverage_micro": 0.012387860877867902 |
| }, |
| "GPQA Raw": { |
| "k1_MAE_OOS_macro": 0.02, |
| "k2_MAE_OOS_macro": 0.0189691, |
| "k3_MAE_OOS_macro": 0.0175445, |
| "k1_signed_coverage_micro": 0.020000000000000014, |
| "k2_signed_coverage_micro": 0.017770189520624313, |
| "k3_signed_coverage_micro": 0.013574371084337357, |
| "mean_MAE_OOS_macro": 0.018837866666666665, |
| "mean_signed_coverage_micro": 0.01711485353498723 |
| }, |
| "IFEval Raw": { |
| "k1_MAE_OOS_macro": 0.048011, |
| "k2_MAE_OOS_macro": 0.0201847, |
| "k3_MAE_OOS_macro": 0.019375, |
| "k1_signed_coverage_micro": -0.0664979345991561, |
| "k2_signed_coverage_micro": -0.007870622073578595, |
| "k3_signed_coverage_micro": 0.019196787148594394, |
| "mean_MAE_OOS_macro": 0.029190233333333333, |
| "mean_signed_coverage_micro": -0.0183905898413801 |
| }, |
| "MATH Lvl 5 Raw": { |
| "k1_MAE_OOS_macro": 0.032517, |
| "k2_MAE_OOS_macro": 0.0178855, |
| "k3_MAE_OOS_macro": 0.0379227, |
| "k1_signed_coverage_micro": -0.04962008227848103, |
| "k2_signed_coverage_micro": -0.012329929765886248, |
| "k3_signed_coverage_micro": -0.03140572771084337, |
| "mean_MAE_OOS_macro": 0.02944173333333333, |
| "mean_signed_coverage_micro": -0.03111857991840355 |
| }, |
| "MMLU-PRO Raw": { |
| "k1_MAE_OOS_macro": 0.0179721, |
| "k2_MAE_OOS_macro": 0.0151516, |
| "k3_MAE_OOS_macro": 0.0170103, |
| "k1_signed_coverage_micro": 0.0052320675105485765, |
| "k2_signed_coverage_micro": 0.014425835005574148, |
| "k3_signed_coverage_micro": 0.016787031325301224, |
| "mean_MAE_OOS_macro": 0.01671133333333333, |
| "mean_signed_coverage_micro": 0.01214831128047465 |
| }, |
| "MUSR Raw": { |
| "k1_MAE_OOS_macro": 0.0183673, |
| "k2_MAE_OOS_macro": 0.0194074, |
| "k3_MAE_OOS_macro": 0.0181618, |
| "k1_signed_coverage_micro": 0.015780727848101264, |
| "k2_signed_coverage_micro": 0.017770347826086975, |
| "k3_signed_coverage_micro": 0.01277109236947792, |
| "mean_MAE_OOS_macro": 0.0186455, |
| "mean_signed_coverage_micro": 0.015440722681222053 |
| } |
| }, |
| "claim5_authors_code": { |
| "MAE_OOS_macro_by_task_alpha": { |
| "BBH Raw": { |
| "5": 0.16658663333333334, |
| "10": 0.07861816666666667, |
| "20": 0.06196636666666666, |
| "50": 0.0471685, |
| "100": 0.0156571 |
| }, |
| "GPQA Raw": { |
| "5": 0.0236493, |
| "10": 0.0199271, |
| "20": 0.017966333333333334, |
| "50": 0.023808866666666668, |
| "100": 0.019059033333333333 |
| }, |
| "IFEval Raw": { |
| "5": 0.15543933333333335, |
| "10": 0.10541753333333333, |
| "20": 0.0680076, |
| "50": 0.04751193333333334, |
| "100": 0.038778 |
| }, |
| "MATH Lvl 5 Raw": { |
| "5": 0.12228146666666667, |
| "10": 0.09935396666666667, |
| "20": 0.09590846666666668, |
| "50": 0.08460853333333333, |
| "100": 0.0558949 |
| }, |
| "MMLU-PRO Raw": { |
| "5": 0.1612238, |
| "10": 0.08756603333333333, |
| "20": 0.0449784, |
| "50": 0.039775066666666664, |
| "100": 0.022609100000000003 |
| }, |
| "MUSR Raw": { |
| "5": 0.029723733333333335, |
| "10": 0.02311313333333333, |
| "20": 0.021194599999999997, |
| "50": 0.020050266666666667, |
| "100": 0.019651533333333332 |
| } |
| }, |
| "macro_mean_by_alpha": { |
| "5": 0.10981737777777778, |
| "10": 0.06899932222222223, |
| "20": 0.05167029444444444, |
| "50": 0.04382052777777778, |
| "100": 0.028608277777777774 |
| } |
| }, |
| "claim6_authors_code": { |
| "X_post_max": 5.109977737428537, |
| "X_post_min": -1.056389658245763, |
| "X_pre_max": 3.3468033095963556, |
| "X_pre_min": -1.8152899666382492, |
| "X_support_hi": 3.3468033095963556, |
| "X_support_lo": -1.056389658245763, |
| "coef_group_shift_gamma": 0.5756989528983756, |
| "coef_intercept_a": -4.278842982074016, |
| "coef_slope_b": 1.3557444173948832, |
| "eps_clip": 1e-06, |
| "gamma_perm_mean": 0.2918169125031601, |
| "gamma_perm_q05": -0.3614995547495127, |
| "gamma_perm_q50": 0.3697808951699048, |
| "gamma_perm_q95": 0.6725590405018136, |
| "gamma_perm_std": 0.31078256611596033, |
| "gamma_perm_zscore": 0.9134426166276405, |
| "n_bins": 10, |
| "n_dropped_outside_support": 25, |
| "n_missing_release_date": 0, |
| "n_perm": 2000, |
| "n_post": 62, |
| "n_post_release": 86, |
| "n_pre": 28, |
| "n_pre_release": 29, |
| "n_total_merged": 115, |
| "n_used_in_support": 90, |
| "n_used_total": 115, |
| "n_with_release_date": 115, |
| "odds_ratio_group_shift": 1.7783730916431397, |
| "pvalue_one_sided_gamma_gt_0": 0.14842578710644677, |
| "pvalue_two_sided": 0.1489255372313843, |
| "r2_logit_fit": 0.5288603684522073, |
| "release_cutoff": "2025-02-06", |
| "se_group_shift_gamma": 0.32613271712611586, |
| "seed": 0, |
| "t_group_shift_gamma": 1.7652290698444468, |
| "test_name": "logit_to_logit_group_shift" |
| } |
| } |
|
|
