| { | |
| "seed": 20260502, | |
| "created": "2026-08-31 11:06:33", | |
| "specialists": { | |
| "sba": { | |
| "uncalibrated_test": { | |
| "auc": 0.98006, | |
| "pr_auc": 0.91569, | |
| "brier": 0.04957, | |
| "ece": 0.06379, | |
| "gini": 0.96011, | |
| "n": 224292, | |
| "base_rate": 0.17561 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.98001, | |
| "pr_auc": 0.91265, | |
| "brier": 0.03779, | |
| "ece": 0.00153, | |
| "gini": 0.96001, | |
| "n": 224292, | |
| "base_rate": 0.17561 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.98031, | |
| "pr_auc": 0.91761, | |
| "brier": 0.04878, | |
| "ece": 0.05966, | |
| "gini": 0.96062, | |
| "n": 224292, | |
| "base_rate": 0.17561 | |
| }, | |
| "xgb": { | |
| "auc": 0.97817, | |
| "pr_auc": 0.9118, | |
| "brier": 0.05141, | |
| "ece": 0.06325, | |
| "gini": 0.95633, | |
| "n": 224292, | |
| "base_rate": 0.17561 | |
| }, | |
| "cat": { | |
| "auc": 0.97941, | |
| "pr_auc": 0.91138, | |
| "brier": 0.05159, | |
| "ece": 0.06845, | |
| "gini": 0.95883, | |
| "n": 224292, | |
| "base_rate": 0.17561 | |
| } | |
| }, | |
| "split": { | |
| "fit": 538300, | |
| "cal": 134575, | |
| "test": 224292 | |
| }, | |
| "train_seconds": 149.3, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "lending_club": { | |
| "uncalibrated_test": { | |
| "auc": 0.72638, | |
| "pr_auc": 0.41511, | |
| "brier": 0.20917, | |
| "ece": 0.23617, | |
| "gini": 0.45276, | |
| "n": 342392, | |
| "base_rate": 0.21235 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.72628, | |
| "pr_auc": 0.41038, | |
| "brier": 0.14842, | |
| "ece": 0.00242, | |
| "gini": 0.45256, | |
| "n": 342392, | |
| "base_rate": 0.21235 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.72571, | |
| "pr_auc": 0.41398, | |
| "brier": 0.20907, | |
| "ece": 0.23511, | |
| "gini": 0.45142, | |
| "n": 342392, | |
| "base_rate": 0.21235 | |
| }, | |
| "xgb": { | |
| "auc": 0.72496, | |
| "pr_auc": 0.41335, | |
| "brier": 0.20779, | |
| "ece": 0.23177, | |
| "gini": 0.44993, | |
| "n": 342392, | |
| "base_rate": 0.21235 | |
| }, | |
| "cat": { | |
| "auc": 0.72527, | |
| "pr_auc": 0.41297, | |
| "brier": 0.21182, | |
| "ece": 0.24162, | |
| "gini": 0.45054, | |
| "n": 342392, | |
| "base_rate": 0.21235 | |
| } | |
| }, | |
| "split": { | |
| "fit": 821739, | |
| "cal": 205435, | |
| "test": 342392 | |
| }, | |
| "train_seconds": 313.2, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "home_credit": { | |
| "uncalibrated_test": { | |
| "auc": 0.76391, | |
| "pr_auc": 0.25504, | |
| "brier": 0.14005, | |
| "ece": 0.2309, | |
| "gini": 0.52782, | |
| "n": 76879, | |
| "base_rate": 0.08072 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.76316, | |
| "pr_auc": 0.24491, | |
| "brier": 0.06737, | |
| "ece": 0.00106, | |
| "gini": 0.52632, | |
| "n": 76879, | |
| "base_rate": 0.08072 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.76161, | |
| "pr_auc": 0.25264, | |
| "brier": 0.13299, | |
| "ece": 0.20947, | |
| "gini": 0.52322, | |
| "n": 76879, | |
| "base_rate": 0.08072 | |
| }, | |
| "xgb": { | |
| "auc": 0.74833, | |
| "pr_auc": 0.2422, | |
| "brier": 0.12857, | |
| "ece": 0.19716, | |
| "gini": 0.49666, | |
| "n": 76879, | |
| "base_rate": 0.08072 | |
| }, | |
| "cat": { | |
| "auc": 0.76386, | |
| "pr_auc": 0.25336, | |
| "brier": 0.17028, | |
| "ece": 0.28606, | |
| "gini": 0.52773, | |
| "n": 76879, | |
| "base_rate": 0.08072 | |
| } | |
| }, | |
| "split": { | |
| "fit": 184506, | |
| "cal": 46126, | |
| "test": 76879 | |
| }, | |
| "train_seconds": 847.8, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "gmsc": { | |
| "uncalibrated_test": { | |
| "auc": 0.85285, | |
| "pr_auc": 0.38106, | |
| "brier": 0.1069, | |
| "ece": 0.17457, | |
| "gini": 0.7057, | |
| "n": 37500, | |
| "base_rate": 0.06683 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.85219, | |
| "pr_auc": 0.36286, | |
| "brier": 0.05019, | |
| "ece": 0.00414, | |
| "gini": 0.70439, | |
| "n": 37500, | |
| "base_rate": 0.06683 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.84924, | |
| "pr_auc": 0.37588, | |
| "brier": 0.10118, | |
| "ece": 0.15231, | |
| "gini": 0.69849, | |
| "n": 37500, | |
| "base_rate": 0.06683 | |
| }, | |
| "xgb": { | |
| "auc": 0.84359, | |
| "pr_auc": 0.363, | |
| "brier": 0.10092, | |
| "ece": 0.15575, | |
| "gini": 0.68719, | |
| "n": 37500, | |
| "base_rate": 0.06683 | |
| }, | |
| "cat": { | |
| "auc": 0.85575, | |
| "pr_auc": 0.38912, | |
| "brier": 0.12636, | |
| "ece": 0.21565, | |
| "gini": 0.7115, | |
| "n": 37500, | |
| "base_rate": 0.06683 | |
| } | |
| }, | |
| "split": { | |
| "fit": 90000, | |
| "cal": 22500, | |
| "test": 37500 | |
| }, | |
| "train_seconds": 30.3, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "taiwan": { | |
| "uncalibrated_test": { | |
| "auc": 0.77195, | |
| "pr_auc": 0.54858, | |
| "brier": 0.1552, | |
| "ece": 0.10805, | |
| "gini": 0.5439, | |
| "n": 7500, | |
| "base_rate": 0.2212 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.76981, | |
| "pr_auc": 0.5243, | |
| "brier": 0.13654, | |
| "ece": 0.00741, | |
| "gini": 0.53962, | |
| "n": 7500, | |
| "base_rate": 0.2212 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.7665, | |
| "pr_auc": 0.53502, | |
| "brier": 0.15551, | |
| "ece": 0.09934, | |
| "gini": 0.533, | |
| "n": 7500, | |
| "base_rate": 0.2212 | |
| }, | |
| "xgb": { | |
| "auc": 0.76662, | |
| "pr_auc": 0.53978, | |
| "brier": 0.15536, | |
| "ece": 0.1021, | |
| "gini": 0.53324, | |
| "n": 7500, | |
| "base_rate": 0.2212 | |
| }, | |
| "cat": { | |
| "auc": 0.7736, | |
| "pr_auc": 0.5542, | |
| "brier": 0.16475, | |
| "ece": 0.14506, | |
| "gini": 0.5472, | |
| "n": 7500, | |
| "base_rate": 0.2212 | |
| } | |
| }, | |
| "split": { | |
| "fit": 18000, | |
| "cal": 4500, | |
| "test": 7500 | |
| }, | |
| "train_seconds": 19.6, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "german": { | |
| "uncalibrated_test": { | |
| "auc": 0.77524, | |
| "pr_auc": 0.62165, | |
| "brier": 0.18357, | |
| "ece": 0.1152, | |
| "gini": 0.55048, | |
| "n": 250, | |
| "base_rate": 0.3 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.76682, | |
| "pr_auc": 0.55053, | |
| "brier": 0.1726, | |
| "ece": 0.068, | |
| "gini": 0.53364, | |
| "n": 250, | |
| "base_rate": 0.3 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.75573, | |
| "pr_auc": 0.60112, | |
| "brier": 0.19909, | |
| "ece": 0.13886, | |
| "gini": 0.51147, | |
| "n": 250, | |
| "base_rate": 0.3 | |
| }, | |
| "xgb": { | |
| "auc": 0.74811, | |
| "pr_auc": 0.58436, | |
| "brier": 0.20829, | |
| "ece": 0.15212, | |
| "gini": 0.49623, | |
| "n": 250, | |
| "base_rate": 0.3 | |
| }, | |
| "cat": { | |
| "auc": 0.79078, | |
| "pr_auc": 0.62009, | |
| "brier": 0.17517, | |
| "ece": 0.1206, | |
| "gini": 0.58156, | |
| "n": 250, | |
| "base_rate": 0.3 | |
| } | |
| }, | |
| "split": { | |
| "fit": 600, | |
| "cal": 150, | |
| "test": 250 | |
| }, | |
| "train_seconds": 11.9, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "berka": { | |
| "uncalibrated_test": { | |
| "auc": 0.87708, | |
| "pr_auc": 0.61967, | |
| "brier": 0.07064, | |
| "ece": 0.05592, | |
| "gini": 0.75416, | |
| "n": 171, | |
| "base_rate": 0.11111 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.85578, | |
| "pr_auc": 0.57253, | |
| "brier": 0.06747, | |
| "ece": 0.02955, | |
| "gini": 0.71157, | |
| "n": 171, | |
| "base_rate": 0.11111 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.88089, | |
| "pr_auc": 0.59759, | |
| "brier": 0.07508, | |
| "ece": 0.06377, | |
| "gini": 0.76177, | |
| "n": 171, | |
| "base_rate": 0.11111 | |
| }, | |
| "xgb": { | |
| "auc": 0.88158, | |
| "pr_auc": 0.651, | |
| "brier": 0.06728, | |
| "ece": 0.06099, | |
| "gini": 0.76316, | |
| "n": 171, | |
| "base_rate": 0.11111 | |
| }, | |
| "cat": { | |
| "auc": 0.82999, | |
| "pr_auc": 0.56659, | |
| "brier": 0.07593, | |
| "ece": 0.06894, | |
| "gini": 0.65997, | |
| "n": 171, | |
| "base_rate": 0.11111 | |
| } | |
| }, | |
| "split": { | |
| "fit": 409, | |
| "cal": 102, | |
| "test": 171 | |
| }, | |
| "train_seconds": 6.8, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "amex": { | |
| "uncalibrated_test": { | |
| "auc": 0.95817, | |
| "pr_auc": 0.89247, | |
| "brier": 0.07574, | |
| "ece": 0.04162, | |
| "gini": 0.91633, | |
| "n": 30000, | |
| "base_rate": 0.25983 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.95796, | |
| "pr_auc": 0.88581, | |
| "brier": 0.07064, | |
| "ece": 0.00502, | |
| "gini": 0.91591, | |
| "n": 30000, | |
| "base_rate": 0.25983 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.95756, | |
| "pr_auc": 0.89073, | |
| "brier": 0.07557, | |
| "ece": 0.03815, | |
| "gini": 0.91512, | |
| "n": 30000, | |
| "base_rate": 0.25983 | |
| }, | |
| "xgb": { | |
| "auc": 0.95721, | |
| "pr_auc": 0.89041, | |
| "brier": 0.07588, | |
| "ece": 0.03907, | |
| "gini": 0.91443, | |
| "n": 30000, | |
| "base_rate": 0.25983 | |
| }, | |
| "cat": { | |
| "auc": 0.95735, | |
| "pr_auc": 0.89028, | |
| "brier": 0.07999, | |
| "ece": 0.05991, | |
| "gini": 0.91469, | |
| "n": 30000, | |
| "base_rate": 0.25983 | |
| } | |
| }, | |
| "split": { | |
| "fit": 72000, | |
| "cal": 18000, | |
| "test": 30000 | |
| }, | |
| "train_seconds": 566.9, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ] | |
| }, | |
| "hc2024": { | |
| "error": "TypeError: object of type 'numpy.bool' has no len()" | |
| } | |
| }, | |
| "pooled": { | |
| "uncalibrated_test": { | |
| "auc": 0.86058, | |
| "pr_auc": 0.58803, | |
| "brier": 0.14928, | |
| "ece": 0.21003, | |
| "gini": 0.72116, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.86047, | |
| "pr_auc": 0.57954, | |
| "brier": 0.08676, | |
| "ece": 0.00176, | |
| "gini": 0.72094, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.86067, | |
| "pr_auc": 0.58929, | |
| "brier": 0.14837, | |
| "ece": 0.20657, | |
| "gini": 0.72135, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "xgb": { | |
| "auc": 0.85863, | |
| "pr_auc": 0.58613, | |
| "brier": 0.14788, | |
| "ece": 0.20508, | |
| "gini": 0.71726, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "cat": { | |
| "auc": 0.85908, | |
| "pr_auc": 0.58117, | |
| "brier": 0.15355, | |
| "ece": 0.21845, | |
| "gini": 0.71815, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| } | |
| }, | |
| "split": { | |
| "fit": 649009, | |
| "cal": 162252, | |
| "test": 270421 | |
| }, | |
| "train_seconds": 154.4, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ], | |
| "datasets": [ | |
| "sba", | |
| "lending_club", | |
| "home_credit", | |
| "gmsc", | |
| "taiwan", | |
| "german", | |
| "berka" | |
| ], | |
| "normalised": true | |
| }, | |
| "lodo": { | |
| "berka": { | |
| "auc": 0.51549, | |
| "pr_auc": 0.11137, | |
| "brier": 0.11782, | |
| "ece": 0.10719, | |
| "gini": 0.03098, | |
| "n": 682, | |
| "base_rate": 0.11144 | |
| }, | |
| "german": { | |
| "auc": 0.40224, | |
| "pr_auc": 0.25219, | |
| "brier": 0.2571, | |
| "ece": 0.19897, | |
| "gini": -0.19552, | |
| "n": 1000, | |
| "base_rate": 0.3 | |
| }, | |
| "gmsc": { | |
| "auc": 0.77113, | |
| "pr_auc": 0.24962, | |
| "brier": 0.08841, | |
| "ece": 0.18062, | |
| "gini": 0.54226, | |
| "n": 150000, | |
| "base_rate": 0.06684 | |
| }, | |
| "home_credit": { | |
| "auc": 0.57854, | |
| "pr_auc": 0.10119, | |
| "brier": 0.11983, | |
| "ece": 0.18925, | |
| "gini": 0.15707, | |
| "n": 300000, | |
| "base_rate": 0.08076 | |
| }, | |
| "lending_club": { | |
| "auc": 0.53986, | |
| "pr_auc": 0.23571, | |
| "brier": 0.18794, | |
| "ece": 0.14079, | |
| "gini": 0.07971, | |
| "n": 300000, | |
| "base_rate": 0.21212 | |
| }, | |
| "sba": { | |
| "auc": 0.25738, | |
| "pr_auc": 0.11727, | |
| "brier": 0.16272, | |
| "ece": 0.15218, | |
| "gini": -0.48525, | |
| "n": 300000, | |
| "base_rate": 0.17512 | |
| }, | |
| "taiwan": { | |
| "auc": 0.69894, | |
| "pr_auc": 0.40759, | |
| "brier": 0.17723, | |
| "ece": 0.14459, | |
| "gini": 0.39788, | |
| "n": 30000, | |
| "base_rate": 0.2212 | |
| } | |
| }, | |
| "notes": { | |
| "berka_leakage_fix": "Berka aggregates ONLY pre-origination transactions. Using the full history leaks the label (71% of transactions post-date the loan) and yields AUC 1.0000, which is not reported." | |
| }, | |
| "pooled_raw": { | |
| "uncalibrated_test": { | |
| "auc": 0.8594, | |
| "pr_auc": 0.58403, | |
| "brier": 0.15075, | |
| "ece": 0.21275, | |
| "gini": 0.7188, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "calibrated_test": { | |
| "auc": 0.85932, | |
| "pr_auc": 0.57674, | |
| "brier": 0.08721, | |
| "ece": 0.00149, | |
| "gini": 0.71864, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "per_member_test": { | |
| "lgbm": { | |
| "auc": 0.86009, | |
| "pr_auc": 0.58794, | |
| "brier": 0.14937, | |
| "ece": 0.20785, | |
| "gini": 0.72019, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "xgb": { | |
| "auc": 0.85282, | |
| "pr_auc": 0.56994, | |
| "brier": 0.15216, | |
| "ece": 0.21138, | |
| "gini": 0.70564, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| }, | |
| "cat": { | |
| "auc": 0.85869, | |
| "pr_auc": 0.58063, | |
| "brier": 0.15401, | |
| "ece": 0.21902, | |
| "gini": 0.71738, | |
| "n": 270421, | |
| "base_rate": 0.14555 | |
| } | |
| }, | |
| "split": { | |
| "fit": 649009, | |
| "cal": 162252, | |
| "test": 270421 | |
| }, | |
| "train_seconds": 134.0, | |
| "members": [ | |
| "lgbm", | |
| "xgb", | |
| "cat" | |
| ], | |
| "datasets": [ | |
| "sba", | |
| "lending_club", | |
| "home_credit", | |
| "gmsc", | |
| "taiwan", | |
| "german", | |
| "berka" | |
| ] | |
| }, | |
| "lodo_raw": { | |
| "berka": { | |
| "auc": 0.46123, | |
| "pr_auc": 0.10863, | |
| "brier": 0.10266, | |
| "ece": 0.04246, | |
| "gini": -0.07754, | |
| "n": 682, | |
| "base_rate": 0.11144 | |
| }, | |
| "german": { | |
| "auc": 0.46458, | |
| "pr_auc": 0.29047, | |
| "brier": 0.23741, | |
| "ece": 0.15139, | |
| "gini": -0.07084, | |
| "n": 1000, | |
| "base_rate": 0.3 | |
| }, | |
| "gmsc": { | |
| "auc": 0.77861, | |
| "pr_auc": 0.28425, | |
| "brier": 0.08594, | |
| "ece": 0.17679, | |
| "gini": 0.55722, | |
| "n": 150000, | |
| "base_rate": 0.06684 | |
| }, | |
| "home_credit": { | |
| "auc": 0.4991, | |
| "pr_auc": 0.07872, | |
| "brier": 0.22521, | |
| "ece": 0.37382, | |
| "gini": -0.0018, | |
| "n": 300000, | |
| "base_rate": 0.08076 | |
| }, | |
| "lending_club": { | |
| "auc": 0.52276, | |
| "pr_auc": 0.22327, | |
| "brier": 0.18042, | |
| "ece": 0.11233, | |
| "gini": 0.04551, | |
| "n": 300000, | |
| "base_rate": 0.21212 | |
| }, | |
| "sba": { | |
| "auc": 0.41031, | |
| "pr_auc": 0.15392, | |
| "brier": 0.17302, | |
| "ece": 0.1638, | |
| "gini": -0.17938, | |
| "n": 300000, | |
| "base_rate": 0.17512 | |
| }, | |
| "taiwan": { | |
| "auc": 0.7038, | |
| "pr_auc": 0.45141, | |
| "brier": 0.15338, | |
| "ece": 0.07019, | |
| "gini": 0.40759, | |
| "n": 30000, | |
| "base_rate": 0.2212 | |
| } | |
| }, | |
| "lodo_mean_auc": 0.5377, | |
| "lodo_raw_mean_auc": 0.5486 | |
| } |