scaled_run/summary.json

{
  "classification": [
    {
      "KL_ERT": 0.07839163582134417,
      "KL_decomposition_residual": 0.0,
      "KL_minus_ERT": 0.038500802761714054,
      "KL_plus_ERT": 0.03989083305963012,
      "L1_ERT": 0.09544,
      "L2_ERT": 0.015467227875312442,
      "base_accuracy": 0.82905,
      "base_epochs": 3,
      "dataset": "FashionMNIST",
      "legacy_sign_KL_ERT": -0.07839163582134417,
      "legacy_sign_KL_minus_ERT": -0.038500802761714054,
      "legacy_sign_KL_plus_ERT": -0.03989083305963012,
      "marginal_coverage": 0.905,
      "mean_set_size": 1.2328,
      "method": "likelihood",
      "n_calibration": 12000,
      "n_test": 20000,
      "n_train": 9000,
      "seed": 0,
      "threshold": -0.23498015105724335
    },
    {
      "KL_ERT": 0.2473487981585557,
      "KL_decomposition_residual": 0.0,
      "KL_minus_ERT": 0.1790285127212846,
      "KL_plus_ERT": 0.06832028543727112,
      "L1_ERT": 0.15654,
      "L2_ERT": 0.07058733514833733,
      "base_accuracy": 0.82905,
      "base_epochs": 3,
      "dataset": "FashionMNIST",
      "legacy_sign_KL_ERT": -0.2473487981585557,
      "legacy_sign_KL_minus_ERT": -0.1790285127212846,
      "legacy_sign_KL_plus_ERT": -0.06832028543727112,
      "marginal_coverage": 0.9029,
      "mean_set_size": 3.2255,
      "method": "cumulative",
      "n_calibration": 12000,
      "n_test": 20000,
      "n_train": 9000,
      "seed": 0,
      "threshold": 0.9983662962913513
    }
  ],
  "crossfit_control": {
    "mean_crossfit_L1_ERT": 0.0012662013958125572,
    "mean_in_sample_L1_ERT": 0.17940179461615158,
    "mean_unweighted_minus_weighted": 3.10208677450016e-06,
    "package_three_point_unweighted_control": 0.5,
    "paper_three_point_weighted_control": 0.3333333333333333,
    "repetitions": 20
  },
  "runtime_seconds": 109.7270896434784,
  "synthetic_at_target": [
    {
      "CovGap": 0.011570376198602982,
      "KL_ERT": -0.022802216118992982,
      "L1_ERT": -0.002780000000000004,
      "L2_ERT": -0.0036128488714285726,
      "classifier": "LightGBM",
      "scenario": "oracle_sets",
      "theory_L1": 0.0,
      "theory_L2": 0.0
    },
    {
      "CovGap": 0.011570376198602982,
      "KL_ERT": -0.0008148918730929743,
      "L1_ERT": 0.0023950000000000017,
      "L2_ERT": -0.00013446674580422694,
      "classifier": "PartitionWise",
      "scenario": "oracle_sets",
      "theory_L1": 0.0,
      "theory_L2": 0.0
    },
    {
      "CovGap": 0.013368118780142362,
      "KL_ERT": 0.04161978778729219,
      "L1_ERT": 0.07701,
      "L2_ERT": 0.005145980945278534,
      "classifier": "LightGBM",
      "scenario": "standard_CP",
      "theory_L1": 0.09264155222114662,
      "theory_L2": 0.01121403975613083
    },
    {
      "CovGap": 0.013368118780142362,
      "KL_ERT": -0.00020618547329738144,
      "L1_ERT": 0.0077,
      "L2_ERT": -2.011247706273581e-05,
      "classifier": "PartitionWise",
      "scenario": "standard_CP",
      "theory_L1": 0.09264155222114662,
      "theory_L2": 0.01121403975613083
    }
  ],
  "target_sample_size": 5000,
  "theory_audit": {
    "KL_ERT": 0.05679002932752115,
    "KL_minus_ERT": 0.05122119560126623,
    "KL_plus_ERT": 0.005568833726254922,
    "KL_residual": 0.0,
    "L1_ERT": 0.08333333333333333,
    "L1_minus_ERT": 0.0666666666666667,
    "L1_plus_ERT": 0.016666666666666646,
    "L1_residual": -1.3877787807814457e-17,
    "L2_ERT": 0.014166666666666675,
    "coverage": 0.9,
    "probabilities": [
      0.7,
      0.9,
      0.95
    ]
  }
}