| { |
| "claim2": { |
| "normalisation": "percent of the largest ERT over all methods AND all numbers of test samples, per dataset and repetition, then averaged (Table 2 caption); negative ERT clipped to 0", |
| "paper_table2_L1": { |
| "CheapBetterLGBMClassifier": 68.4, |
| "PartitionWise": 38.3, |
| "BetterCatBoost": 68.7, |
| "RF": 65.9, |
| "XT": 65.9, |
| "tabICL": 71.9, |
| "TabPFN": 71.6 |
| }, |
| "variants": { |
| "v1 (results_old.csv)": { |
| "rows": 2800, |
| "L1_percent_of_max": { |
| "CheapBetterLGBMClassifier": 68.03638140127894, |
| "XT": 65.52544651991043, |
| "RF": 65.39950342632956, |
| "tabICL": 71.5102160122718, |
| "PartitionWise": 37.8207068825266, |
| "TabPFN": 71.23955924079642, |
| "BetterCatBoost": 68.66960392756035 |
| }, |
| "lightgbm": 68.03638140127894, |
| "partitionwise": 37.8207068825266, |
| "lightgbm_abs_err_vs_paper": 0.36361859872106095, |
| "partitionwise_abs_err_vs_paper": 0.47929311747339653, |
| "ordering_lightgbm_above_partitionwise": true |
| }, |
| "v2 (results.csv)": { |
| "rows": 5600, |
| "L1_percent_of_max": { |
| "BetterCatBoost": 70.31627805977823, |
| "CheapBetterLGBMClassifier": 68.92900479082506, |
| "XT": 66.53143321909444, |
| "RF": 67.2767890259383, |
| "tabICL": 72.66607482837067, |
| "PartitionWise": 33.670548352764385, |
| "TabPFN": 72.13568950693384 |
| }, |
| "lightgbm": 68.92900479082506, |
| "partitionwise": 33.670548352764385, |
| "lightgbm_abs_err_vs_paper": 0.5290047908250557, |
| "partitionwise_abs_err_vs_paper": 4.629451647235612, |
| "ordering_lightgbm_above_partitionwise": true |
| } |
| }, |
| "other_metrics_v2": { |
| "L2": { |
| "BetterCatBoost": 49.0071922409777, |
| "CheapBetterLGBMClassifier": 46.435713430066265, |
| "XT": 30.302556102139143, |
| "RF": 36.87651319709627, |
| "tabICL": 53.9583162881316, |
| "PartitionWise": 10.530135474406409, |
| "TabPFN": 49.748553099895055 |
| }, |
| "KL": { |
| "BetterCatBoost": 54.05819133012458, |
| "CheapBetterLGBMClassifier": 50.935353447465104, |
| "XT": 27.670695250125288, |
| "RF": 35.0531315347866, |
| "tabICL": 59.246746583269534, |
| "PartitionWise": 11.012516952290087, |
| "TabPFN": 55.22562054445726 |
| } |
| } |
| }, |
| "claim5": { |
| "n_rows": 80, |
| "decomposition_max_residual": 8.940696721859709e-08, |
| "per_dataset_method": { |
| "MNIST|likelihood": { |
| "KL": [ |
| -0.18710587758504865, |
| 0.006658818037739759 |
| ], |
| "KL_plus": [ |
| -0.12831058113152882, |
| 0.005195663786398997 |
| ], |
| "KL_minus": [ |
| -0.05879531943273254, |
| 0.0024970636507724605 |
| ] |
| }, |
| "MNIST|cumulative": { |
| "KL": [ |
| -0.21624940017182154, |
| 0.01666774016886198 |
| ], |
| "KL_plus": [ |
| -0.15939179494622038, |
| 0.01161647543368673 |
| ], |
| "KL_minus": [ |
| -0.05685761234871077, |
| 0.005610714577396073 |
| ] |
| }, |
| "FashionMNIST|likelihood": { |
| "KL": [ |
| -0.06811388991773122, |
| 0.007993181559542269 |
| ], |
| "KL_plus": [ |
| -0.04212642349302763, |
| 0.005677110447665139 |
| ], |
| "KL_minus": [ |
| -0.02598746716976159, |
| 0.004767266311988138 |
| ] |
| }, |
| "FashionMNIST|cumulative": { |
| "KL": [ |
| -0.2601063385605812, |
| 0.013834332625820572 |
| ], |
| "KL_plus": [ |
| -0.18529993444681164, |
| 0.010265827845468862 |
| ], |
| "KL_minus": [ |
| -0.07480641230940815, |
| 0.003877947026407775 |
| ] |
| }, |
| "CIFAR10|likelihood": { |
| "KL": [ |
| 0.02832810527194678, |
| 0.00618225335103705 |
| ], |
| "KL_plus": [ |
| 0.006766273948223831, |
| 0.001416814412288527 |
| ], |
| "KL_minus": [ |
| 0.02156181978604231, |
| 0.007053554145809787 |
| ] |
| }, |
| "CIFAR10|cumulative": { |
| "KL": [ |
| -0.017296617239686982, |
| 0.007835687912677925 |
| ], |
| "KL_plus": [ |
| -0.02951636482503982, |
| 0.005342355728146586 |
| ], |
| "KL_minus": [ |
| 0.01221973485697023, |
| 0.0061652773637478776 |
| ] |
| }, |
| "CIFAR100|likelihood": { |
| "KL": [ |
| 0.4086424125659942, |
| 0.02541961045936785 |
| ], |
| "KL_plus": [ |
| 0.08522980659590909, |
| 0.021734443672694895 |
| ], |
| "KL_minus": [ |
| 0.3234126213317108, |
| 0.01978366305585833 |
| ] |
| }, |
| "CIFAR100|cumulative": { |
| "KL": [ |
| 0.1905506563616333, |
| 0.024133401088240462 |
| ], |
| "KL_plus": [ |
| 0.016023607273191902, |
| 0.007820076340537635 |
| ], |
| "KL_minus": [ |
| 0.1745270594703445, |
| 0.026338485382953438 |
| ] |
| } |
| }, |
| "n_cells": 8, |
| "n_cells_with_divergent_components": 8, |
| "naming": "the released columns are 'underconfident'/'overconfident'; an underconfident predictor makes sets that are too wide, i.e. over-coverage, which is the paper's l_+ component" |
| }, |
| "claim1": { |
| "alpha": 0.1, |
| "d": 8, |
| "n_test": 5000, |
| "n_cal": 3000, |
| "seeds": 5, |
| "standard": { |
| "L1": 0.063272, |
| "true_L1": 0.0702964914948382, |
| "L2": 0.005256268125075593, |
| "true_L2": 0.007167963207236776, |
| "KL": 0.030604667681682535, |
| "true_KL": 0.0416029844930876, |
| "marginal_coverage": 0.89856 |
| }, |
| "oracle": { |
| "L1": 0.0011679999999999982, |
| "true_L1": 0.0, |
| "L2": -0.00017319047147024304, |
| "true_L2": 0.0, |
| "KL": -0.0010198970862035728, |
| "true_KL": 0.0, |
| "marginal_coverage": 0.90032 |
| }, |
| "oracle_L1_near_zero": true, |
| "standard_L1_positive": true, |
| "separation_L1": 0.062104 |
| }, |
| "claim6": { |
| "true": 0.0690035899522276, |
| "in_sample_tree": 0.1743, |
| "cv_tree": 0.021024999999999995, |
| "cv_hgb": 0.05347500000000001, |
| "oracle_in_sample_tree": 0.17670000000000002, |
| "oracle_cv_tree": 0.004424999999999995, |
| "n_test": 1000, |
| "in_sample_inflation_factor": 8.290130796670633, |
| "oracle_in_sample_false_positive": 0.17670000000000002, |
| "oracle_cv_is_near_zero": true |
| }, |
| "claim4": { |
| "n_test": 6000, |
| "seeds": 8, |
| "scenarios": { |
| "conservative (radius x1.35)": { |
| "L1_plus_est": 0.06408333333333331, |
| "L1_minus_est": 0.0010625000000000014, |
| "L1_plus_true": 0.06608391764460793, |
| "L1_minus_true": 0.003515496941181244, |
| "KL_plus_est": 0.044628850516583406, |
| "KL_minus_est": 0.00021369709520476876, |
| "KL_plus_true": 0.051062027112734255, |
| "KL_minus_true": 0.0011484626555137195, |
| "marginal_coverage": 0.9630208333333333, |
| "scale": 1.35, |
| "L1_ratio_plus_over_minus": 60.31372549019598, |
| "dominant_side_est": "plus", |
| "dominant_side_true": "plus", |
| "side_attributed_correctly": true |
| }, |
| "aggressive (radius x0.75)": { |
| "L1_plus_est": 0.008629166666666663, |
| "L1_minus_est": 0.11244166666666666, |
| "L1_plus_true": 0.012156084075637029, |
| "L1_minus_true": 0.11614065781185745, |
| "KL_plus_est": 0.0021941574834898414, |
| "KL_minus_est": 0.08249845596638003, |
| "KL_plus_true": 0.006623309264618229, |
| "KL_minus_true": 0.09067184559363284, |
| "marginal_coverage": 0.7961875, |
| "scale": 0.75, |
| "L1_ratio_plus_over_minus": 0.0767434966278811, |
| "dominant_side_est": "minus", |
| "dominant_side_true": "minus", |
| "side_attributed_correctly": true |
| }, |
| "standard (radius x1)": { |
| "L1_plus_est": 0.029933333333333322, |
| "L1_minus_est": 0.03262083333333334, |
| "L1_plus_true": 0.03334231184161376, |
| "L1_minus_true": 0.03734773786332873, |
| "KL_plus_est": 0.01555761941505708, |
| "KL_minus_est": 0.013312671224877568, |
| "KL_plus_true": 0.02204298832902235, |
| "KL_minus_true": 0.01938098116174198, |
| "marginal_coverage": 0.8973125000000001, |
| "scale": 1.0, |
| "L1_ratio_plus_over_minus": 0.917613999233618, |
| "dominant_side_est": "minus", |
| "dominant_side_true": "minus", |
| "side_attributed_correctly": true |
| } |
| }, |
| "all_sides_attributed_correctly": true, |
| "separation_conservative_vs_aggressive": 785.913180144099 |
| }, |
| "claim3": { |
| "rows": [ |
| { |
| "ert_std": 0.03576, |
| "ert_ora": -0.0016800000000000014, |
| "cg_std": 0.031072781526658745, |
| "cg_ora": 0.031199778923785264, |
| "true_std": 0.06880447557795312, |
| "n_test": 500, |
| "ert_error_vs_true": 0.03304447557795312, |
| "covgap_error_vs_true": 0.037731694051294375, |
| "ert_separation": 0.03744, |
| "covgap_separation": -0.0001269973971265194 |
| }, |
| { |
| "ert_std": 0.05286, |
| "ert_ora": -0.0035400000000000015, |
| "cg_std": 0.025883724155466782, |
| "cg_ora": 0.024059037379189192, |
| "true_std": 0.06889486259000807, |
| "n_test": 1000, |
| "ert_error_vs_true": 0.016034862590008074, |
| "covgap_error_vs_true": 0.04301113843454129, |
| "ert_separation": 0.0564, |
| "covgap_separation": 0.0018246867762775898 |
| }, |
| { |
| "ert_std": 0.05676000000000001, |
| "ert_ora": -0.002210000000000001, |
| "cg_std": 0.018405934181208283, |
| "cg_ora": 0.014972773544179094, |
| "true_std": 0.06954455557994782, |
| "n_test": 2000, |
| "ert_error_vs_true": 0.012784555579947808, |
| "covgap_error_vs_true": 0.05113862139873954, |
| "ert_separation": 0.058970000000000015, |
| "covgap_separation": 0.003433160637029189 |
| }, |
| { |
| "ert_std": 0.06160800000000001, |
| "ert_ora": -0.0005080000000000013, |
| "cg_std": 0.014439326900402834, |
| "cg_ora": 0.010073670837015896, |
| "true_std": 0.06944134707377626, |
| "n_test": 5000, |
| "ert_error_vs_true": 0.007833347073776253, |
| "covgap_error_vs_true": 0.055002020173373425, |
| "ert_separation": 0.06211600000000001, |
| "covgap_separation": 0.004365656063386939 |
| } |
| ], |
| "n_seeds": 10, |
| "ert_closer_to_truth_at_all_n": true, |
| "separation_ratio_at_max_n": 14.228331114066169 |
| } |
| } |
