outputs/claim1_pvalue_validity.json

{
  "seed0": 20260725,
  "A_oracle_uniformity": {
    "n30|wilcoxon|perm": {
      "n_pvalues": 2000,
      "ks_stat": 0.03875000000000001,
      "ks_pvalue": 0.004794504795142653,
      "mean_pvalue": 0.513717440456152,
      "type_I_at_0.05": 0.04,
      "type_I_at_0.10": 0.0875,
      "type_I_at_0.20": 0.1805,
      "seconds": 7.6
    },
    "n30|wilcoxon|iid": {
      "n_pvalues": 2000,
      "ks_stat": 0.055499999999999994,
      "ks_pvalue": 8.529640795381222e-06,
      "mean_pvalue": 0.5321838897168636,
      "type_I_at_0.05": 0.04,
      "type_I_at_0.10": 0.0825,
      "type_I_at_0.20": 0.1715,
      "seconds": 0.7
    },
    "n30|sign|perm": {
      "n_pvalues": 2000,
      "ks_stat": 0.1849999999999999,
      "ks_pvalue": 2.178061515066372e-60,
      "mean_pvalue": 0.6011963584423066,
      "type_I_at_0.05": 0.0205,
      "type_I_at_0.10": 0.0485,
      "type_I_at_0.20": 0.1245,
      "seconds": 0.7
    },
    "n30|sign|iid": {
      "n_pvalues": 2000,
      "ks_stat": 0.1939999999999999,
      "ks_pvalue": 2.0563954874359258e-66,
      "mean_pvalue": 0.5974072098135949,
      "type_I_at_0.05": 0.0235,
      "type_I_at_0.10": 0.0495,
      "type_I_at_0.20": 0.131,
      "seconds": 0.7
    },
    "n100|wilcoxon|perm": {
      "n_pvalues": 2000,
      "ks_stat": 0.031593749999999976,
      "ks_pvalue": 0.03611565396799554,
      "mean_pvalue": 0.5142353996248362,
      "type_I_at_0.05": 0.053,
      "type_I_at_0.10": 0.0895,
      "type_I_at_0.20": 0.18,
      "seconds": 1.6
    },
    "n100|wilcoxon|iid": {
      "n_pvalues": 2000,
      "ks_stat": 0.017171875000000003,
      "ks_pvalue": 0.5908896295949981,
      "mean_pvalue": 0.5040204703577134,
      "type_I_at_0.05": 0.0545,
      "type_I_at_0.10": 0.1075,
      "type_I_at_0.20": 0.1985,
      "seconds": 1.6
    },
    "n100|sign|perm": {
      "n_pvalues": 2000,
      "ks_stat": 0.11549999999999916,
      "ks_pvalue": 1.0628268177294982e-23,
      "mean_pvalue": 0.5567143145029297,
      "type_I_at_0.05": 0.03,
      "type_I_at_0.10": 0.063,
      "type_I_at_0.20": 0.146,
      "seconds": 1.8
    },
    "n100|sign|iid": {
      "n_pvalues": 2000,
      "ks_stat": 0.13549999999999918,
      "ks_pvalue": 1.732641730975926e-32,
      "mean_pvalue": 0.5705219966314959,
      "type_I_at_0.05": 0.0325,
      "type_I_at_0.10": 0.068,
      "type_I_at_0.20": 0.148,
      "seconds": 2.4
    },
    "n300|wilcoxon|perm": {
      "n_pvalues": 2000,
      "ks_stat": 0.020796877984483375,
      "ks_pvalue": 0.3478159210597468,
      "mean_pvalue": 0.5063172186678839,
      "type_I_at_0.05": 0.0445,
      "type_I_at_0.10": 0.101,
      "type_I_at_0.20": 0.201,
      "seconds": 7.1
    },
    "n300|wilcoxon|iid": {
      "n_pvalues": 2000,
      "ks_stat": 0.027628676570077948,
      "ks_pvalue": 0.0926498459791032,
      "mean_pvalue": 0.511867285252981,
      "type_I_at_0.05": 0.0525,
      "type_I_at_0.10": 0.096,
      "type_I_at_0.20": 0.1845,
      "seconds": 7.5
    },
    "n300|sign|perm": {
      "n_pvalues": 2000,
      "ks_stat": 0.08649999999999919,
      "ks_pvalue": 1.810003764044721e-13,
      "mean_pvalue": 0.54722458389146,
      "type_I_at_0.05": 0.038,
      "type_I_at_0.10": 0.0835,
      "type_I_at_0.20": 0.168,
      "seconds": 6.0
    },
    "n300|sign|iid": {
      "n_pvalues": 2000,
      "ks_stat": 0.0914999999999992,
      "ks_pvalue": 5.067852085315274e-15,
      "mean_pvalue": 0.5362474015107654,
      "type_I_at_0.05": 0.035,
      "type_I_at_0.10": 0.0755,
      "type_I_at_0.20": 0.1725,
      "seconds": 6.0
    }
  },
  "B_split_vs_nosplit": {
    "R2_with_split_heldout": 0.8543329698449388,
    "n_null_pvalues": 1200,
    "replicates": 100,
    "seconds": 28.1,
    "SKO_Wcx_nosplit": {
      "n_pvalues": 1200,
      "ks_stat": 0.026510416666666647,
      "ks_pvalue": 0.3615977605796692,
      "mean_pvalue": 0.5054051943343442,
      "type_I_at_0.05": 0.0525,
      "type_I_at_0.10": 0.10583333333333333,
      "type_I_at_0.20": 0.20166666666666666
    },
    "HRT_nosplit": {
      "n_pvalues": 1200,
      "ks_stat": 0.9151492537313433,
      "ks_pvalue": 0.0,
      "mean_pvalue": 0.02322139303482587,
      "type_I_at_0.05": 0.9525,
      "type_I_at_0.10": 0.9666666666666667,
      "type_I_at_0.20": 0.9791666666666666
    },
    "HRT_split": {
      "n_pvalues": 1200,
      "ks_stat": 0.06726368159203977,
      "ks_pvalue": 3.6461620675348725e-05,
      "mean_pvalue": 0.5096019900497513,
      "type_I_at_0.05": 0.08416666666666667,
      "type_I_at_0.10": 0.13416666666666666,
      "type_I_at_0.20": 0.22916666666666666
    },
    "HRT_split_oracle_sampler": {
      "n_pvalues": 1200,
      "ks_stat": 0.028445273631840795,
      "ks_pvalue": 0.28068834940853593,
      "mean_pvalue": 0.5044485903814262,
      "type_I_at_0.05": 0.04083333333333333,
      "type_I_at_0.10": 0.09666666666666666,
      "type_I_at_0.20": 0.205
    }
  },
  "C_heavy_tailed_elliptical": {
    "design": "multivariate t(3), AR(1) rho=0.6",
    "replicates": 200,
    "seconds": 16.6,
    "SKO_Wcx_oracle_nu": {
      "n_pvalues": 2000,
      "ks_stat": 0.018403628253797433,
      "ks_pvalue": 0.5013216498021769,
      "mean_pvalue": 0.5076241390269205,
      "type_I_at_0.05": 0.0405,
      "type_I_at_0.10": 0.0905,
      "type_I_at_0.20": 0.1835
    },
    "HRT_split_gaussian_sampler": {
      "n_pvalues": 2000,
      "ks_stat": 0.11147263681592035,
      "ks_pvalue": 4.2131554211688007e-22,
      "mean_pvalue": 0.5131890547263681,
      "type_I_at_0.05": 0.13,
      "type_I_at_0.10": 0.1845,
      "type_I_at_0.20": 0.269
    }
  }
}