| { |
| "analysis": "pre-committed repeated scaffold-disjoint split robustness check for Delta R2", |
| "gate": { |
| "checks": { |
| "delta_r2": { |
| "abs_error": 0.0, |
| "expected": 0.15844577805518212, |
| "observed": 0.15844577805518212, |
| "passes": true, |
| "tolerance": 0.002 |
| }, |
| "rho_base": { |
| "abs_error": 0.0, |
| "expected": 0.6130893034049266, |
| "observed": 0.6130893034049266, |
| "passes": true, |
| "tolerance": 0.002 |
| }, |
| "rho_struct": { |
| "abs_error": 0.0, |
| "expected": 0.6633801733693918, |
| "observed": 0.6633801733693918, |
| "passes": true, |
| "tolerance": 0.002 |
| } |
| }, |
| "seed": 20260710, |
| "status": "pass" |
| }, |
| "model_specs": { |
| "baseline": "baseline_physical_gbm", |
| "fixed_hyperparameters": { |
| "imputer": "SimpleImputer(strategy='median')", |
| "l2_regularization": 0.1, |
| "learning_rate": 0.04, |
| "max_iter": 200, |
| "model": "HistGradientBoostingRegressor", |
| "random_state": 17 |
| }, |
| "source": "scripts/train_substantivity.py fit_predict(model_name='gbm')", |
| "structure": "structure_full_gbm" |
| }, |
| "n_model_ready": 170, |
| "pimt_version": "v8", |
| "seed_list_precommitted": [ |
| 20260710, |
| 20260711, |
| 20260712, |
| 20260713, |
| 20260714, |
| 20260715, |
| 20260716, |
| 20260717, |
| 20260718, |
| 20260719, |
| 20260720, |
| 20260721, |
| 20260722, |
| 20260723, |
| 20260724, |
| 20260725, |
| 20260726, |
| 20260727, |
| 20260728, |
| 20260729 |
| ], |
| "splits": [ |
| { |
| "delta_r2": 0.15844577805518212, |
| "delta_rho": 0.05029086996446519, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.24057748815493107, |
| "r2_struct": 0.3990232662101132, |
| "rho_base": 0.6130893034049266, |
| "rho_struct": 0.6633801733693918, |
| "scaffold_overlap": [], |
| "seed": 20260710, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.10894368939423504, |
| "delta_rho": -0.02142567109834309, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.0753173569506469, |
| "r2_struct": 0.18426104634488194, |
| "rho_base": 0.3374255928820426, |
| "rho_struct": 0.3159999217836995, |
| "scaffold_overlap": [], |
| "seed": 20260711, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": false, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.07229900764275499, |
| "delta_rho": 0.054794018070953965, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.03474908347621042, |
| "r2_struct": 0.1070480911189654, |
| "rho_base": 0.3116758043837438, |
| "rho_struct": 0.3664698224546978, |
| "scaffold_overlap": [], |
| "seed": 20260712, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": -0.07932888055232179, |
| "delta_rho": -0.18782560792479713, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.009371529716526661, |
| "r2_struct": -0.06995735083579513, |
| "rho_base": 0.22892683918665618, |
| "rho_struct": 0.04110123126185904, |
| "scaffold_overlap": [], |
| "seed": 20260713, |
| "structure_r2_gt_baseline_r2": false, |
| "structure_rho_gt_baseline_rho": false, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.17795748043035797, |
| "delta_rho": 0.15765396639828383, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": -0.025625114290944717, |
| "r2_struct": 0.15233236613941326, |
| "rho_base": 0.3148971774546397, |
| "rho_struct": 0.4725511438529235, |
| "scaffold_overlap": [], |
| "seed": 20260714, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.3651105688046139, |
| "delta_rho": 0.5723722889991607, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": -0.14781647665809938, |
| "r2_struct": 0.21729409214651452, |
| "rho_base": -0.05458199713591954, |
| "rho_struct": 0.5177902918632412, |
| "scaffold_overlap": [], |
| "seed": 20260715, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.1593569048675234, |
| "delta_rho": 0.26000963937387966, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": -0.31410214728889585, |
| "r2_struct": -0.15474524242137244, |
| "rho_base": 0.021809248206604608, |
| "rho_struct": 0.2818188875804843, |
| "scaffold_overlap": [], |
| "seed": 20260716, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.18827443498189356, |
| "delta_rho": 0.03148922432055268, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": -0.10861175558073333, |
| "r2_struct": 0.07966267940116023, |
| "rho_base": 0.2930425572131179, |
| "rho_struct": 0.32453178153367057, |
| "scaffold_overlap": [], |
| "seed": 20260717, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.06412327813812702, |
| "delta_rho": 0.04651519519605235, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.009225601013266416, |
| "r2_struct": 0.07334887915139343, |
| "rho_base": 0.20013954931621014, |
| "rho_struct": 0.2466547445122625, |
| "scaffold_overlap": [], |
| "seed": 20260718, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.11669355822914607, |
| "delta_rho": 0.2230183252778165, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.11300301261267753, |
| "r2_struct": 0.2296965708418236, |
| "rho_base": 0.29368400613333734, |
| "rho_struct": 0.5167023314111538, |
| "scaffold_overlap": [], |
| "seed": 20260719, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.1615774244680297, |
| "delta_rho": 0.1556327648303413, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": -0.0004697438215104377, |
| "r2_struct": 0.16110768064651926, |
| "rho_base": 0.2151720464246368, |
| "rho_struct": 0.3708048112549781, |
| "scaffold_overlap": [], |
| "seed": 20260720, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.13393454244880798, |
| "delta_rho": 0.011281368170483141, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.12810201588115622, |
| "r2_struct": 0.2620365583299642, |
| "rho_base": 0.305951850465261, |
| "rho_struct": 0.31723321863574416, |
| "scaffold_overlap": [], |
| "seed": 20260721, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.1394537063219743, |
| "delta_rho": 0.10944078731825996, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.1597453729206536, |
| "r2_struct": 0.2991990792426279, |
| "rho_base": 0.4096488396211531, |
| "rho_struct": 0.5190896269394131, |
| "scaffold_overlap": [], |
| "seed": 20260722, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.03749212355150777, |
| "delta_rho": -0.020460924496539412, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.22792739917286775, |
| "r2_struct": 0.2654195227243755, |
| "rho_base": 0.364431504041778, |
| "rho_struct": 0.3439705795452386, |
| "scaffold_overlap": [], |
| "seed": 20260723, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": false, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.027406566812759103, |
| "delta_rho": 0.1042355837894588, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.0790457059498223, |
| "r2_struct": 0.1064522727625814, |
| "rho_base": 0.25539472324636275, |
| "rho_struct": 0.35963030703582155, |
| "scaffold_overlap": [], |
| "seed": 20260724, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.18287532371958493, |
| "delta_rho": 0.1886042686910458, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": -0.001996189735690379, |
| "r2_struct": 0.18087913398389455, |
| "rho_base": 0.22568044935707093, |
| "rho_struct": 0.41428471804811673, |
| "scaffold_overlap": [], |
| "seed": 20260725, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.2023991030263118, |
| "delta_rho": 0.3463187369730992, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.043956322103222534, |
| "r2_struct": 0.24635542512953434, |
| "rho_base": 0.25719833423563027, |
| "rho_struct": 0.6035170712087294, |
| "scaffold_overlap": [], |
| "seed": 20260726, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.05346556978823658, |
| "delta_rho": -0.037189855887579226, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.2499476956003236, |
| "r2_struct": 0.30341326538856017, |
| "rho_base": 0.5136834171866699, |
| "rho_struct": 0.47649356129909065, |
| "scaffold_overlap": [], |
| "seed": 20260727, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": false, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.16469614036100466, |
| "delta_rho": 0.1832511429802354, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.057282262855799515, |
| "r2_struct": 0.22197840321680418, |
| "rho_base": 0.2097134137159253, |
| "rho_struct": 0.3929645566961607, |
| "scaffold_overlap": [], |
| "seed": 20260728, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| }, |
| { |
| "delta_r2": 0.12786896886633592, |
| "delta_rho": 0.24574302338227827, |
| "n_test": 34, |
| "n_train": 136, |
| "r2_base": 0.17693494806547427, |
| "r2_struct": 0.3048039169318102, |
| "rho_base": 0.3036752525270744, |
| "rho_struct": 0.5494182759093527, |
| "scaffold_overlap": [], |
| "seed": 20260729, |
| "structure_r2_gt_baseline_r2": true, |
| "structure_rho_gt_baseline_rho": true, |
| "test_scaffolds": 34, |
| "train_scaffolds": 49 |
| } |
| ], |
| "summary": { |
| "delta_r2": { |
| "fraction_wins": 0.95, |
| "mean": 0.12815226446780326, |
| "n_splits": 20, |
| "percentile_2_5_97_5": [ |
| -0.028629543053908364, |
| 0.2878226225599202 |
| ], |
| "wins": 19 |
| }, |
| "delta_rho": { |
| "fraction_wins": 0.8, |
| "mean": 0.1236874572164554, |
| "n_splits": 20, |
| "percentile_2_5_97_5": [ |
| -0.11627362570711862, |
| 0.4649968517867812 |
| ], |
| "wins": 16 |
| } |
| }, |
| "target": "Poucher measured log10 coefficient only; no inferred, OpenPOM, or volatility-derived substantivity labels used.", |
| "test_fraction": 0.2 |
| } |
|
|
