{
  "provenance": "Re-run of the authors' released scripts (github.com/hlzhang109/prescriptive-scaling) on the authors' released tables, on this machine.",
  "table1_authors_code": {
    "BBH Raw": 0.6995598819257327,
    "GPQA Raw": 0.4244877017896367,
    "IFEval Raw": 0.827845230576459,
    "MATH Lvl 5 Raw": 0.5395001305416097,
    "MMLU-PRO Raw": 0.5626259737685013,
    "MUSR Raw": 0.535245164843684
  },
  "table2_authors_code": {
    "Constant": {
      "pinball_ID": 0.005352683799141467,
      "pinball_OOD": 0.006233912544681612,
      "calib_ID": 0.04123274010277254,
      "calib_OOD": 0.03597267501958549
    },
    "Binwise": {
      "pinball_ID": 0.004014274443578322,
      "pinball_OOD": 0.004995430505610828,
      "calib_ID": 0.016646429776157354,
      "calib_OOD": 0.028066352913576547
    },
    "I-spline": {
      "pinball_ID": 0.004003277885706522,
      "pinball_OOD": 0.004918276548778966,
      "calib_ID": 0.018259079529548153,
      "calib_OOD": 0.024059338259677387
    },
    "Sigmoid": {
      "pinball_ID": 0.004082793246777277,
      "pinball_OOD": 0.00492990898200165,
      "calib_ID": 0.018373223589866268,
      "calib_OOD": 0.02205737092377908
    }
  },
  "n_task_period_pairs": 18,
  "claim4_authors_code": {
    "BBH Raw": {
      "k1_MAE_OOS_macro": 0.0162621,
      "k2_MAE_OOS_macro": 0.0157885,
      "k3_MAE_OOS_macro": 0.0109922,
      "k1_signed_coverage_micro": 0.013671086497890289,
      "k2_signed_coverage_micro": 0.01554063991081382,
      "k3_signed_coverage_micro": 0.007951856224899597,
      "mean_MAE_OOS_macro": 0.0143476,
      "mean_signed_coverage_micro": 0.012387860877867902
    },
    "GPQA Raw": {
      "k1_MAE_OOS_macro": 0.02,
      "k2_MAE_OOS_macro": 0.0189691,
      "k3_MAE_OOS_macro": 0.0175445,
      "k1_signed_coverage_micro": 0.020000000000000014,
      "k2_signed_coverage_micro": 0.017770189520624313,
      "k3_signed_coverage_micro": 0.013574371084337357,
      "mean_MAE_OOS_macro": 0.018837866666666665,
      "mean_signed_coverage_micro": 0.01711485353498723
    },
    "IFEval Raw": {
      "k1_MAE_OOS_macro": 0.048011,
      "k2_MAE_OOS_macro": 0.0201847,
      "k3_MAE_OOS_macro": 0.019375,
      "k1_signed_coverage_micro": -0.0664979345991561,
      "k2_signed_coverage_micro": -0.007870622073578595,
      "k3_signed_coverage_micro": 0.019196787148594394,
      "mean_MAE_OOS_macro": 0.029190233333333333,
      "mean_signed_coverage_micro": -0.0183905898413801
    },
    "MATH Lvl 5 Raw": {
      "k1_MAE_OOS_macro": 0.032517,
      "k2_MAE_OOS_macro": 0.0178855,
      "k3_MAE_OOS_macro": 0.0379227,
      "k1_signed_coverage_micro": -0.04962008227848103,
      "k2_signed_coverage_micro": -0.012329929765886248,
      "k3_signed_coverage_micro": -0.03140572771084337,
      "mean_MAE_OOS_macro": 0.02944173333333333,
      "mean_signed_coverage_micro": -0.03111857991840355
    },
    "MMLU-PRO Raw": {
      "k1_MAE_OOS_macro": 0.0179721,
      "k2_MAE_OOS_macro": 0.0151516,
      "k3_MAE_OOS_macro": 0.0170103,
      "k1_signed_coverage_micro": 0.0052320675105485765,
      "k2_signed_coverage_micro": 0.014425835005574148,
      "k3_signed_coverage_micro": 0.016787031325301224,
      "mean_MAE_OOS_macro": 0.01671133333333333,
      "mean_signed_coverage_micro": 0.01214831128047465
    },
    "MUSR Raw": {
      "k1_MAE_OOS_macro": 0.0183673,
      "k2_MAE_OOS_macro": 0.0194074,
      "k3_MAE_OOS_macro": 0.0181618,
      "k1_signed_coverage_micro": 0.015780727848101264,
      "k2_signed_coverage_micro": 0.017770347826086975,
      "k3_signed_coverage_micro": 0.01277109236947792,
      "mean_MAE_OOS_macro": 0.0186455,
      "mean_signed_coverage_micro": 0.015440722681222053
    }
  },
  "claim5_authors_code": {
    "MAE_OOS_macro_by_task_alpha": {
      "BBH Raw": {
        "5": 0.16658663333333334,
        "10": 0.07861816666666667,
        "20": 0.06196636666666666,
        "50": 0.0471685,
        "100": 0.0156571
      },
      "GPQA Raw": {
        "5": 0.0236493,
        "10": 0.0199271,
        "20": 0.017966333333333334,
        "50": 0.023808866666666668,
        "100": 0.019059033333333333
      },
      "IFEval Raw": {
        "5": 0.15543933333333335,
        "10": 0.10541753333333333,
        "20": 0.0680076,
        "50": 0.04751193333333334,
        "100": 0.038778
      },
      "MATH Lvl 5 Raw": {
        "5": 0.12228146666666667,
        "10": 0.09935396666666667,
        "20": 0.09590846666666668,
        "50": 0.08460853333333333,
        "100": 0.0558949
      },
      "MMLU-PRO Raw": {
        "5": 0.1612238,
        "10": 0.08756603333333333,
        "20": 0.0449784,
        "50": 0.039775066666666664,
        "100": 0.022609100000000003
      },
      "MUSR Raw": {
        "5": 0.029723733333333335,
        "10": 0.02311313333333333,
        "20": 0.021194599999999997,
        "50": 0.020050266666666667,
        "100": 0.019651533333333332
      }
    },
    "macro_mean_by_alpha": {
      "5": 0.10981737777777778,
      "10": 0.06899932222222223,
      "20": 0.05167029444444444,
      "50": 0.04382052777777778,
      "100": 0.028608277777777774
    }
  },
  "claim6_authors_code": {
    "X_post_max": 5.109977737428537,
    "X_post_min": -1.056389658245763,
    "X_pre_max": 3.3468033095963556,
    "X_pre_min": -1.8152899666382492,
    "X_support_hi": 3.3468033095963556,
    "X_support_lo": -1.056389658245763,
    "coef_group_shift_gamma": 0.5756989528983756,
    "coef_intercept_a": -4.278842982074016,
    "coef_slope_b": 1.3557444173948832,
    "eps_clip": 1e-06,
    "gamma_perm_mean": 0.2918169125031601,
    "gamma_perm_q05": -0.3614995547495127,
    "gamma_perm_q50": 0.3697808951699048,
    "gamma_perm_q95": 0.6725590405018136,
    "gamma_perm_std": 0.31078256611596033,
    "gamma_perm_zscore": 0.9134426166276405,
    "n_bins": 10,
    "n_dropped_outside_support": 25,
    "n_missing_release_date": 0,
    "n_perm": 2000,
    "n_post": 62,
    "n_post_release": 86,
    "n_pre": 28,
    "n_pre_release": 29,
    "n_total_merged": 115,
    "n_used_in_support": 90,
    "n_used_total": 115,
    "n_with_release_date": 115,
    "odds_ratio_group_shift": 1.7783730916431397,
    "pvalue_one_sided_gamma_gt_0": 0.14842578710644677,
    "pvalue_two_sided": 0.1489255372313843,
    "r2_logit_fit": 0.5288603684522073,
    "release_cutoff": "2025-02-06",
    "se_group_shift_gamma": 0.32613271712611586,
    "seed": 0,
    "t_group_shift_gamma": 1.7652290698444468,
    "test_name": "logit_to_logit_group_shift"
  }
}