{
  "analysis": "pre-committed repeated scaffold-disjoint split robustness check for Delta R2",
  "gate": {
    "checks": {
      "delta_r2": {
        "abs_error": 0.0,
        "expected": 0.15844577805518212,
        "observed": 0.15844577805518212,
        "passes": true,
        "tolerance": 0.002
      },
      "rho_base": {
        "abs_error": 0.0,
        "expected": 0.6130893034049266,
        "observed": 0.6130893034049266,
        "passes": true,
        "tolerance": 0.002
      },
      "rho_struct": {
        "abs_error": 0.0,
        "expected": 0.6633801733693918,
        "observed": 0.6633801733693918,
        "passes": true,
        "tolerance": 0.002
      }
    },
    "seed": 20260710,
    "status": "pass"
  },
  "model_specs": {
    "baseline": "baseline_physical_gbm",
    "fixed_hyperparameters": {
      "imputer": "SimpleImputer(strategy='median')",
      "l2_regularization": 0.1,
      "learning_rate": 0.04,
      "max_iter": 200,
      "model": "HistGradientBoostingRegressor",
      "random_state": 17
    },
    "source": "scripts/train_substantivity.py fit_predict(model_name='gbm')",
    "structure": "structure_full_gbm"
  },
  "n_model_ready": 170,
  "pimt_version": "v8",
  "seed_list_precommitted": [
    20260710,
    20260711,
    20260712,
    20260713,
    20260714,
    20260715,
    20260716,
    20260717,
    20260718,
    20260719,
    20260720,
    20260721,
    20260722,
    20260723,
    20260724,
    20260725,
    20260726,
    20260727,
    20260728,
    20260729
  ],
  "splits": [
    {
      "delta_r2": 0.15844577805518212,
      "delta_rho": 0.05029086996446519,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.24057748815493107,
      "r2_struct": 0.3990232662101132,
      "rho_base": 0.6130893034049266,
      "rho_struct": 0.6633801733693918,
      "scaffold_overlap": [],
      "seed": 20260710,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.10894368939423504,
      "delta_rho": -0.02142567109834309,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.0753173569506469,
      "r2_struct": 0.18426104634488194,
      "rho_base": 0.3374255928820426,
      "rho_struct": 0.3159999217836995,
      "scaffold_overlap": [],
      "seed": 20260711,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": false,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.07229900764275499,
      "delta_rho": 0.054794018070953965,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.03474908347621042,
      "r2_struct": 0.1070480911189654,
      "rho_base": 0.3116758043837438,
      "rho_struct": 0.3664698224546978,
      "scaffold_overlap": [],
      "seed": 20260712,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": -0.07932888055232179,
      "delta_rho": -0.18782560792479713,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.009371529716526661,
      "r2_struct": -0.06995735083579513,
      "rho_base": 0.22892683918665618,
      "rho_struct": 0.04110123126185904,
      "scaffold_overlap": [],
      "seed": 20260713,
      "structure_r2_gt_baseline_r2": false,
      "structure_rho_gt_baseline_rho": false,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.17795748043035797,
      "delta_rho": 0.15765396639828383,
      "n_test": 34,
      "n_train": 136,
      "r2_base": -0.025625114290944717,
      "r2_struct": 0.15233236613941326,
      "rho_base": 0.3148971774546397,
      "rho_struct": 0.4725511438529235,
      "scaffold_overlap": [],
      "seed": 20260714,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.3651105688046139,
      "delta_rho": 0.5723722889991607,
      "n_test": 34,
      "n_train": 136,
      "r2_base": -0.14781647665809938,
      "r2_struct": 0.21729409214651452,
      "rho_base": -0.05458199713591954,
      "rho_struct": 0.5177902918632412,
      "scaffold_overlap": [],
      "seed": 20260715,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.1593569048675234,
      "delta_rho": 0.26000963937387966,
      "n_test": 34,
      "n_train": 136,
      "r2_base": -0.31410214728889585,
      "r2_struct": -0.15474524242137244,
      "rho_base": 0.021809248206604608,
      "rho_struct": 0.2818188875804843,
      "scaffold_overlap": [],
      "seed": 20260716,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.18827443498189356,
      "delta_rho": 0.03148922432055268,
      "n_test": 34,
      "n_train": 136,
      "r2_base": -0.10861175558073333,
      "r2_struct": 0.07966267940116023,
      "rho_base": 0.2930425572131179,
      "rho_struct": 0.32453178153367057,
      "scaffold_overlap": [],
      "seed": 20260717,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.06412327813812702,
      "delta_rho": 0.04651519519605235,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.009225601013266416,
      "r2_struct": 0.07334887915139343,
      "rho_base": 0.20013954931621014,
      "rho_struct": 0.2466547445122625,
      "scaffold_overlap": [],
      "seed": 20260718,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.11669355822914607,
      "delta_rho": 0.2230183252778165,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.11300301261267753,
      "r2_struct": 0.2296965708418236,
      "rho_base": 0.29368400613333734,
      "rho_struct": 0.5167023314111538,
      "scaffold_overlap": [],
      "seed": 20260719,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.1615774244680297,
      "delta_rho": 0.1556327648303413,
      "n_test": 34,
      "n_train": 136,
      "r2_base": -0.0004697438215104377,
      "r2_struct": 0.16110768064651926,
      "rho_base": 0.2151720464246368,
      "rho_struct": 0.3708048112549781,
      "scaffold_overlap": [],
      "seed": 20260720,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.13393454244880798,
      "delta_rho": 0.011281368170483141,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.12810201588115622,
      "r2_struct": 0.2620365583299642,
      "rho_base": 0.305951850465261,
      "rho_struct": 0.31723321863574416,
      "scaffold_overlap": [],
      "seed": 20260721,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.1394537063219743,
      "delta_rho": 0.10944078731825996,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.1597453729206536,
      "r2_struct": 0.2991990792426279,
      "rho_base": 0.4096488396211531,
      "rho_struct": 0.5190896269394131,
      "scaffold_overlap": [],
      "seed": 20260722,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.03749212355150777,
      "delta_rho": -0.020460924496539412,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.22792739917286775,
      "r2_struct": 0.2654195227243755,
      "rho_base": 0.364431504041778,
      "rho_struct": 0.3439705795452386,
      "scaffold_overlap": [],
      "seed": 20260723,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": false,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.027406566812759103,
      "delta_rho": 0.1042355837894588,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.0790457059498223,
      "r2_struct": 0.1064522727625814,
      "rho_base": 0.25539472324636275,
      "rho_struct": 0.35963030703582155,
      "scaffold_overlap": [],
      "seed": 20260724,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.18287532371958493,
      "delta_rho": 0.1886042686910458,
      "n_test": 34,
      "n_train": 136,
      "r2_base": -0.001996189735690379,
      "r2_struct": 0.18087913398389455,
      "rho_base": 0.22568044935707093,
      "rho_struct": 0.41428471804811673,
      "scaffold_overlap": [],
      "seed": 20260725,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.2023991030263118,
      "delta_rho": 0.3463187369730992,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.043956322103222534,
      "r2_struct": 0.24635542512953434,
      "rho_base": 0.25719833423563027,
      "rho_struct": 0.6035170712087294,
      "scaffold_overlap": [],
      "seed": 20260726,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.05346556978823658,
      "delta_rho": -0.037189855887579226,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.2499476956003236,
      "r2_struct": 0.30341326538856017,
      "rho_base": 0.5136834171866699,
      "rho_struct": 0.47649356129909065,
      "scaffold_overlap": [],
      "seed": 20260727,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": false,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.16469614036100466,
      "delta_rho": 0.1832511429802354,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.057282262855799515,
      "r2_struct": 0.22197840321680418,
      "rho_base": 0.2097134137159253,
      "rho_struct": 0.3929645566961607,
      "scaffold_overlap": [],
      "seed": 20260728,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    },
    {
      "delta_r2": 0.12786896886633592,
      "delta_rho": 0.24574302338227827,
      "n_test": 34,
      "n_train": 136,
      "r2_base": 0.17693494806547427,
      "r2_struct": 0.3048039169318102,
      "rho_base": 0.3036752525270744,
      "rho_struct": 0.5494182759093527,
      "scaffold_overlap": [],
      "seed": 20260729,
      "structure_r2_gt_baseline_r2": true,
      "structure_rho_gt_baseline_rho": true,
      "test_scaffolds": 34,
      "train_scaffolds": 49
    }
  ],
  "summary": {
    "delta_r2": {
      "fraction_wins": 0.95,
      "mean": 0.12815226446780326,
      "n_splits": 20,
      "percentile_2_5_97_5": [
        -0.028629543053908364,
        0.2878226225599202
      ],
      "wins": 19
    },
    "delta_rho": {
      "fraction_wins": 0.8,
      "mean": 0.1236874572164554,
      "n_splits": 20,
      "percentile_2_5_97_5": [
        -0.11627362570711862,
        0.4649968517867812
      ],
      "wins": 16
    }
  },
  "target": "Poucher measured log10 coefficient only; no inferred, OpenPOM, or volatility-derived substantivity labels used.",
  "test_fraction": 0.2
}