{
  "claim2": {
    "normalisation": "percent of the largest ERT over all methods AND all numbers of test samples, per dataset and repetition, then averaged (Table 2 caption); negative ERT clipped to 0",
    "paper_table2_L1": {
      "CheapBetterLGBMClassifier": 68.4,
      "PartitionWise": 38.3,
      "BetterCatBoost": 68.7,
      "RF": 65.9,
      "XT": 65.9,
      "tabICL": 71.9,
      "TabPFN": 71.6
    },
    "variants": {
      "v1 (results_old.csv)": {
        "rows": 2800,
        "L1_percent_of_max": {
          "CheapBetterLGBMClassifier": 68.03638140127894,
          "XT": 65.52544651991043,
          "RF": 65.39950342632956,
          "tabICL": 71.5102160122718,
          "PartitionWise": 37.8207068825266,
          "TabPFN": 71.23955924079642,
          "BetterCatBoost": 68.66960392756035
        },
        "lightgbm": 68.03638140127894,
        "partitionwise": 37.8207068825266,
        "lightgbm_abs_err_vs_paper": 0.36361859872106095,
        "partitionwise_abs_err_vs_paper": 0.47929311747339653,
        "ordering_lightgbm_above_partitionwise": true
      },
      "v2 (results.csv)": {
        "rows": 5600,
        "L1_percent_of_max": {
          "BetterCatBoost": 70.31627805977823,
          "CheapBetterLGBMClassifier": 68.92900479082506,
          "XT": 66.53143321909444,
          "RF": 67.2767890259383,
          "tabICL": 72.66607482837067,
          "PartitionWise": 33.670548352764385,
          "TabPFN": 72.13568950693384
        },
        "lightgbm": 68.92900479082506,
        "partitionwise": 33.670548352764385,
        "lightgbm_abs_err_vs_paper": 0.5290047908250557,
        "partitionwise_abs_err_vs_paper": 4.629451647235612,
        "ordering_lightgbm_above_partitionwise": true
      }
    },
    "other_metrics_v2": {
      "L2": {
        "BetterCatBoost": 49.0071922409777,
        "CheapBetterLGBMClassifier": 46.435713430066265,
        "XT": 30.302556102139143,
        "RF": 36.87651319709627,
        "tabICL": 53.9583162881316,
        "PartitionWise": 10.530135474406409,
        "TabPFN": 49.748553099895055
      },
      "KL": {
        "BetterCatBoost": 54.05819133012458,
        "CheapBetterLGBMClassifier": 50.935353447465104,
        "XT": 27.670695250125288,
        "RF": 35.0531315347866,
        "tabICL": 59.246746583269534,
        "PartitionWise": 11.012516952290087,
        "TabPFN": 55.22562054445726
      }
    }
  },
  "claim5": {
    "n_rows": 80,
    "decomposition_max_residual": 8.940696721859709e-08,
    "per_dataset_method": {
      "MNIST|likelihood": {
        "KL": [
          -0.18710587758504865,
          0.006658818037739759
        ],
        "KL_plus": [
          -0.12831058113152882,
          0.005195663786398997
        ],
        "KL_minus": [
          -0.05879531943273254,
          0.0024970636507724605
        ]
      },
      "MNIST|cumulative": {
        "KL": [
          -0.21624940017182154,
          0.01666774016886198
        ],
        "KL_plus": [
          -0.15939179494622038,
          0.01161647543368673
        ],
        "KL_minus": [
          -0.05685761234871077,
          0.005610714577396073
        ]
      },
      "FashionMNIST|likelihood": {
        "KL": [
          -0.06811388991773122,
          0.007993181559542269
        ],
        "KL_plus": [
          -0.04212642349302763,
          0.005677110447665139
        ],
        "KL_minus": [
          -0.02598746716976159,
          0.004767266311988138
        ]
      },
      "FashionMNIST|cumulative": {
        "KL": [
          -0.2601063385605812,
          0.013834332625820572
        ],
        "KL_plus": [
          -0.18529993444681164,
          0.010265827845468862
        ],
        "KL_minus": [
          -0.07480641230940815,
          0.003877947026407775
        ]
      },
      "CIFAR10|likelihood": {
        "KL": [
          0.02832810527194678,
          0.00618225335103705
        ],
        "KL_plus": [
          0.006766273948223831,
          0.001416814412288527
        ],
        "KL_minus": [
          0.02156181978604231,
          0.007053554145809787
        ]
      },
      "CIFAR10|cumulative": {
        "KL": [
          -0.017296617239686982,
          0.007835687912677925
        ],
        "KL_plus": [
          -0.02951636482503982,
          0.005342355728146586
        ],
        "KL_minus": [
          0.01221973485697023,
          0.0061652773637478776
        ]
      },
      "CIFAR100|likelihood": {
        "KL": [
          0.4086424125659942,
          0.02541961045936785
        ],
        "KL_plus": [
          0.08522980659590909,
          0.021734443672694895
        ],
        "KL_minus": [
          0.3234126213317108,
          0.01978366305585833
        ]
      },
      "CIFAR100|cumulative": {
        "KL": [
          0.1905506563616333,
          0.024133401088240462
        ],
        "KL_plus": [
          0.016023607273191902,
          0.007820076340537635
        ],
        "KL_minus": [
          0.1745270594703445,
          0.026338485382953438
        ]
      }
    },
    "n_cells": 8,
    "n_cells_with_divergent_components": 8,
    "naming": "the released columns are 'underconfident'/'overconfident'; an underconfident predictor makes sets that are too wide, i.e. over-coverage, which is the paper's l_+ component"
  },
  "claim1": {
    "alpha": 0.1,
    "d": 8,
    "n_test": 5000,
    "n_cal": 3000,
    "seeds": 5,
    "standard": {
      "L1": 0.063272,
      "true_L1": 0.0702964914948382,
      "L2": 0.005256268125075593,
      "true_L2": 0.007167963207236776,
      "KL": 0.030604667681682535,
      "true_KL": 0.0416029844930876,
      "marginal_coverage": 0.89856
    },
    "oracle": {
      "L1": 0.0011679999999999982,
      "true_L1": 0.0,
      "L2": -0.00017319047147024304,
      "true_L2": 0.0,
      "KL": -0.0010198970862035728,
      "true_KL": 0.0,
      "marginal_coverage": 0.90032
    },
    "oracle_L1_near_zero": true,
    "standard_L1_positive": true,
    "separation_L1": 0.062104
  },
  "claim6": {
    "true": 0.0690035899522276,
    "in_sample_tree": 0.1743,
    "cv_tree": 0.021024999999999995,
    "cv_hgb": 0.05347500000000001,
    "oracle_in_sample_tree": 0.17670000000000002,
    "oracle_cv_tree": 0.004424999999999995,
    "n_test": 1000,
    "in_sample_inflation_factor": 8.290130796670633,
    "oracle_in_sample_false_positive": 0.17670000000000002,
    "oracle_cv_is_near_zero": true
  },
  "claim4": {
    "n_test": 6000,
    "seeds": 8,
    "scenarios": {
      "conservative (radius x1.35)": {
        "L1_plus_est": 0.06408333333333331,
        "L1_minus_est": 0.0010625000000000014,
        "L1_plus_true": 0.06608391764460793,
        "L1_minus_true": 0.003515496941181244,
        "KL_plus_est": 0.044628850516583406,
        "KL_minus_est": 0.00021369709520476876,
        "KL_plus_true": 0.051062027112734255,
        "KL_minus_true": 0.0011484626555137195,
        "marginal_coverage": 0.9630208333333333,
        "scale": 1.35,
        "L1_ratio_plus_over_minus": 60.31372549019598,
        "dominant_side_est": "plus",
        "dominant_side_true": "plus",
        "side_attributed_correctly": true
      },
      "aggressive (radius x0.75)": {
        "L1_plus_est": 0.008629166666666663,
        "L1_minus_est": 0.11244166666666666,
        "L1_plus_true": 0.012156084075637029,
        "L1_minus_true": 0.11614065781185745,
        "KL_plus_est": 0.0021941574834898414,
        "KL_minus_est": 0.08249845596638003,
        "KL_plus_true": 0.006623309264618229,
        "KL_minus_true": 0.09067184559363284,
        "marginal_coverage": 0.7961875,
        "scale": 0.75,
        "L1_ratio_plus_over_minus": 0.0767434966278811,
        "dominant_side_est": "minus",
        "dominant_side_true": "minus",
        "side_attributed_correctly": true
      },
      "standard (radius x1)": {
        "L1_plus_est": 0.029933333333333322,
        "L1_minus_est": 0.03262083333333334,
        "L1_plus_true": 0.03334231184161376,
        "L1_minus_true": 0.03734773786332873,
        "KL_plus_est": 0.01555761941505708,
        "KL_minus_est": 0.013312671224877568,
        "KL_plus_true": 0.02204298832902235,
        "KL_minus_true": 0.01938098116174198,
        "marginal_coverage": 0.8973125000000001,
        "scale": 1.0,
        "L1_ratio_plus_over_minus": 0.917613999233618,
        "dominant_side_est": "minus",
        "dominant_side_true": "minus",
        "side_attributed_correctly": true
      }
    },
    "all_sides_attributed_correctly": true,
    "separation_conservative_vs_aggressive": 785.913180144099
  },
  "claim3": {
    "rows": [
      {
        "ert_std": 0.03576,
        "ert_ora": -0.0016800000000000014,
        "cg_std": 0.031072781526658745,
        "cg_ora": 0.031199778923785264,
        "true_std": 0.06880447557795312,
        "n_test": 500,
        "ert_error_vs_true": 0.03304447557795312,
        "covgap_error_vs_true": 0.037731694051294375,
        "ert_separation": 0.03744,
        "covgap_separation": -0.0001269973971265194
      },
      {
        "ert_std": 0.05286,
        "ert_ora": -0.0035400000000000015,
        "cg_std": 0.025883724155466782,
        "cg_ora": 0.024059037379189192,
        "true_std": 0.06889486259000807,
        "n_test": 1000,
        "ert_error_vs_true": 0.016034862590008074,
        "covgap_error_vs_true": 0.04301113843454129,
        "ert_separation": 0.0564,
        "covgap_separation": 0.0018246867762775898
      },
      {
        "ert_std": 0.05676000000000001,
        "ert_ora": -0.002210000000000001,
        "cg_std": 0.018405934181208283,
        "cg_ora": 0.014972773544179094,
        "true_std": 0.06954455557994782,
        "n_test": 2000,
        "ert_error_vs_true": 0.012784555579947808,
        "covgap_error_vs_true": 0.05113862139873954,
        "ert_separation": 0.058970000000000015,
        "covgap_separation": 0.003433160637029189
      },
      {
        "ert_std": 0.06160800000000001,
        "ert_ora": -0.0005080000000000013,
        "cg_std": 0.014439326900402834,
        "cg_ora": 0.010073670837015896,
        "true_std": 0.06944134707377626,
        "n_test": 5000,
        "ert_error_vs_true": 0.007833347073776253,
        "covgap_error_vs_true": 0.055002020173373425,
        "ert_separation": 0.06211600000000001,
        "covgap_separation": 0.004365656063386939
      }
    ],
    "n_seeds": 10,
    "ert_closer_to_truth_at_all_n": true,
    "separation_ratio_at_max_n": 14.228331114066169
  }
}