{
  "seed": 20260502,
  "created": "2026-08-31 11:06:33",
  "specialists": {
    "sba": {
      "uncalibrated_test": {
        "auc": 0.98006,
        "pr_auc": 0.91569,
        "brier": 0.04957,
        "ece": 0.06379,
        "gini": 0.96011,
        "n": 224292,
        "base_rate": 0.17561
      },
      "calibrated_test": {
        "auc": 0.98001,
        "pr_auc": 0.91265,
        "brier": 0.03779,
        "ece": 0.00153,
        "gini": 0.96001,
        "n": 224292,
        "base_rate": 0.17561
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.98031,
          "pr_auc": 0.91761,
          "brier": 0.04878,
          "ece": 0.05966,
          "gini": 0.96062,
          "n": 224292,
          "base_rate": 0.17561
        },
        "xgb": {
          "auc": 0.97817,
          "pr_auc": 0.9118,
          "brier": 0.05141,
          "ece": 0.06325,
          "gini": 0.95633,
          "n": 224292,
          "base_rate": 0.17561
        },
        "cat": {
          "auc": 0.97941,
          "pr_auc": 0.91138,
          "brier": 0.05159,
          "ece": 0.06845,
          "gini": 0.95883,
          "n": 224292,
          "base_rate": 0.17561
        }
      },
      "split": {
        "fit": 538300,
        "cal": 134575,
        "test": 224292
      },
      "train_seconds": 149.3,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "lending_club": {
      "uncalibrated_test": {
        "auc": 0.72638,
        "pr_auc": 0.41511,
        "brier": 0.20917,
        "ece": 0.23617,
        "gini": 0.45276,
        "n": 342392,
        "base_rate": 0.21235
      },
      "calibrated_test": {
        "auc": 0.72628,
        "pr_auc": 0.41038,
        "brier": 0.14842,
        "ece": 0.00242,
        "gini": 0.45256,
        "n": 342392,
        "base_rate": 0.21235
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.72571,
          "pr_auc": 0.41398,
          "brier": 0.20907,
          "ece": 0.23511,
          "gini": 0.45142,
          "n": 342392,
          "base_rate": 0.21235
        },
        "xgb": {
          "auc": 0.72496,
          "pr_auc": 0.41335,
          "brier": 0.20779,
          "ece": 0.23177,
          "gini": 0.44993,
          "n": 342392,
          "base_rate": 0.21235
        },
        "cat": {
          "auc": 0.72527,
          "pr_auc": 0.41297,
          "brier": 0.21182,
          "ece": 0.24162,
          "gini": 0.45054,
          "n": 342392,
          "base_rate": 0.21235
        }
      },
      "split": {
        "fit": 821739,
        "cal": 205435,
        "test": 342392
      },
      "train_seconds": 313.2,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "home_credit": {
      "uncalibrated_test": {
        "auc": 0.76391,
        "pr_auc": 0.25504,
        "brier": 0.14005,
        "ece": 0.2309,
        "gini": 0.52782,
        "n": 76879,
        "base_rate": 0.08072
      },
      "calibrated_test": {
        "auc": 0.76316,
        "pr_auc": 0.24491,
        "brier": 0.06737,
        "ece": 0.00106,
        "gini": 0.52632,
        "n": 76879,
        "base_rate": 0.08072
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.76161,
          "pr_auc": 0.25264,
          "brier": 0.13299,
          "ece": 0.20947,
          "gini": 0.52322,
          "n": 76879,
          "base_rate": 0.08072
        },
        "xgb": {
          "auc": 0.74833,
          "pr_auc": 0.2422,
          "brier": 0.12857,
          "ece": 0.19716,
          "gini": 0.49666,
          "n": 76879,
          "base_rate": 0.08072
        },
        "cat": {
          "auc": 0.76386,
          "pr_auc": 0.25336,
          "brier": 0.17028,
          "ece": 0.28606,
          "gini": 0.52773,
          "n": 76879,
          "base_rate": 0.08072
        }
      },
      "split": {
        "fit": 184506,
        "cal": 46126,
        "test": 76879
      },
      "train_seconds": 847.8,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "gmsc": {
      "uncalibrated_test": {
        "auc": 0.85285,
        "pr_auc": 0.38106,
        "brier": 0.1069,
        "ece": 0.17457,
        "gini": 0.7057,
        "n": 37500,
        "base_rate": 0.06683
      },
      "calibrated_test": {
        "auc": 0.85219,
        "pr_auc": 0.36286,
        "brier": 0.05019,
        "ece": 0.00414,
        "gini": 0.70439,
        "n": 37500,
        "base_rate": 0.06683
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.84924,
          "pr_auc": 0.37588,
          "brier": 0.10118,
          "ece": 0.15231,
          "gini": 0.69849,
          "n": 37500,
          "base_rate": 0.06683
        },
        "xgb": {
          "auc": 0.84359,
          "pr_auc": 0.363,
          "brier": 0.10092,
          "ece": 0.15575,
          "gini": 0.68719,
          "n": 37500,
          "base_rate": 0.06683
        },
        "cat": {
          "auc": 0.85575,
          "pr_auc": 0.38912,
          "brier": 0.12636,
          "ece": 0.21565,
          "gini": 0.7115,
          "n": 37500,
          "base_rate": 0.06683
        }
      },
      "split": {
        "fit": 90000,
        "cal": 22500,
        "test": 37500
      },
      "train_seconds": 30.3,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "taiwan": {
      "uncalibrated_test": {
        "auc": 0.77195,
        "pr_auc": 0.54858,
        "brier": 0.1552,
        "ece": 0.10805,
        "gini": 0.5439,
        "n": 7500,
        "base_rate": 0.2212
      },
      "calibrated_test": {
        "auc": 0.76981,
        "pr_auc": 0.5243,
        "brier": 0.13654,
        "ece": 0.00741,
        "gini": 0.53962,
        "n": 7500,
        "base_rate": 0.2212
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.7665,
          "pr_auc": 0.53502,
          "brier": 0.15551,
          "ece": 0.09934,
          "gini": 0.533,
          "n": 7500,
          "base_rate": 0.2212
        },
        "xgb": {
          "auc": 0.76662,
          "pr_auc": 0.53978,
          "brier": 0.15536,
          "ece": 0.1021,
          "gini": 0.53324,
          "n": 7500,
          "base_rate": 0.2212
        },
        "cat": {
          "auc": 0.7736,
          "pr_auc": 0.5542,
          "brier": 0.16475,
          "ece": 0.14506,
          "gini": 0.5472,
          "n": 7500,
          "base_rate": 0.2212
        }
      },
      "split": {
        "fit": 18000,
        "cal": 4500,
        "test": 7500
      },
      "train_seconds": 19.6,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "german": {
      "uncalibrated_test": {
        "auc": 0.77524,
        "pr_auc": 0.62165,
        "brier": 0.18357,
        "ece": 0.1152,
        "gini": 0.55048,
        "n": 250,
        "base_rate": 0.3
      },
      "calibrated_test": {
        "auc": 0.76682,
        "pr_auc": 0.55053,
        "brier": 0.1726,
        "ece": 0.068,
        "gini": 0.53364,
        "n": 250,
        "base_rate": 0.3
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.75573,
          "pr_auc": 0.60112,
          "brier": 0.19909,
          "ece": 0.13886,
          "gini": 0.51147,
          "n": 250,
          "base_rate": 0.3
        },
        "xgb": {
          "auc": 0.74811,
          "pr_auc": 0.58436,
          "brier": 0.20829,
          "ece": 0.15212,
          "gini": 0.49623,
          "n": 250,
          "base_rate": 0.3
        },
        "cat": {
          "auc": 0.79078,
          "pr_auc": 0.62009,
          "brier": 0.17517,
          "ece": 0.1206,
          "gini": 0.58156,
          "n": 250,
          "base_rate": 0.3
        }
      },
      "split": {
        "fit": 600,
        "cal": 150,
        "test": 250
      },
      "train_seconds": 11.9,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "berka": {
      "uncalibrated_test": {
        "auc": 0.87708,
        "pr_auc": 0.61967,
        "brier": 0.07064,
        "ece": 0.05592,
        "gini": 0.75416,
        "n": 171,
        "base_rate": 0.11111
      },
      "calibrated_test": {
        "auc": 0.85578,
        "pr_auc": 0.57253,
        "brier": 0.06747,
        "ece": 0.02955,
        "gini": 0.71157,
        "n": 171,
        "base_rate": 0.11111
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.88089,
          "pr_auc": 0.59759,
          "brier": 0.07508,
          "ece": 0.06377,
          "gini": 0.76177,
          "n": 171,
          "base_rate": 0.11111
        },
        "xgb": {
          "auc": 0.88158,
          "pr_auc": 0.651,
          "brier": 0.06728,
          "ece": 0.06099,
          "gini": 0.76316,
          "n": 171,
          "base_rate": 0.11111
        },
        "cat": {
          "auc": 0.82999,
          "pr_auc": 0.56659,
          "brier": 0.07593,
          "ece": 0.06894,
          "gini": 0.65997,
          "n": 171,
          "base_rate": 0.11111
        }
      },
      "split": {
        "fit": 409,
        "cal": 102,
        "test": 171
      },
      "train_seconds": 6.8,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "amex": {
      "uncalibrated_test": {
        "auc": 0.95817,
        "pr_auc": 0.89247,
        "brier": 0.07574,
        "ece": 0.04162,
        "gini": 0.91633,
        "n": 30000,
        "base_rate": 0.25983
      },
      "calibrated_test": {
        "auc": 0.95796,
        "pr_auc": 0.88581,
        "brier": 0.07064,
        "ece": 0.00502,
        "gini": 0.91591,
        "n": 30000,
        "base_rate": 0.25983
      },
      "per_member_test": {
        "lgbm": {
          "auc": 0.95756,
          "pr_auc": 0.89073,
          "brier": 0.07557,
          "ece": 0.03815,
          "gini": 0.91512,
          "n": 30000,
          "base_rate": 0.25983
        },
        "xgb": {
          "auc": 0.95721,
          "pr_auc": 0.89041,
          "brier": 0.07588,
          "ece": 0.03907,
          "gini": 0.91443,
          "n": 30000,
          "base_rate": 0.25983
        },
        "cat": {
          "auc": 0.95735,
          "pr_auc": 0.89028,
          "brier": 0.07999,
          "ece": 0.05991,
          "gini": 0.91469,
          "n": 30000,
          "base_rate": 0.25983
        }
      },
      "split": {
        "fit": 72000,
        "cal": 18000,
        "test": 30000
      },
      "train_seconds": 566.9,
      "members": [
        "lgbm",
        "xgb",
        "cat"
      ]
    },
    "hc2024": {
      "error": "TypeError: object of type 'numpy.bool' has no len()"
    }
  },
  "pooled": {
    "uncalibrated_test": {
      "auc": 0.86058,
      "pr_auc": 0.58803,
      "brier": 0.14928,
      "ece": 0.21003,
      "gini": 0.72116,
      "n": 270421,
      "base_rate": 0.14555
    },
    "calibrated_test": {
      "auc": 0.86047,
      "pr_auc": 0.57954,
      "brier": 0.08676,
      "ece": 0.00176,
      "gini": 0.72094,
      "n": 270421,
      "base_rate": 0.14555
    },
    "per_member_test": {
      "lgbm": {
        "auc": 0.86067,
        "pr_auc": 0.58929,
        "brier": 0.14837,
        "ece": 0.20657,
        "gini": 0.72135,
        "n": 270421,
        "base_rate": 0.14555
      },
      "xgb": {
        "auc": 0.85863,
        "pr_auc": 0.58613,
        "brier": 0.14788,
        "ece": 0.20508,
        "gini": 0.71726,
        "n": 270421,
        "base_rate": 0.14555
      },
      "cat": {
        "auc": 0.85908,
        "pr_auc": 0.58117,
        "brier": 0.15355,
        "ece": 0.21845,
        "gini": 0.71815,
        "n": 270421,
        "base_rate": 0.14555
      }
    },
    "split": {
      "fit": 649009,
      "cal": 162252,
      "test": 270421
    },
    "train_seconds": 154.4,
    "members": [
      "lgbm",
      "xgb",
      "cat"
    ],
    "datasets": [
      "sba",
      "lending_club",
      "home_credit",
      "gmsc",
      "taiwan",
      "german",
      "berka"
    ],
    "normalised": true
  },
  "lodo": {
    "berka": {
      "auc": 0.51549,
      "pr_auc": 0.11137,
      "brier": 0.11782,
      "ece": 0.10719,
      "gini": 0.03098,
      "n": 682,
      "base_rate": 0.11144
    },
    "german": {
      "auc": 0.40224,
      "pr_auc": 0.25219,
      "brier": 0.2571,
      "ece": 0.19897,
      "gini": -0.19552,
      "n": 1000,
      "base_rate": 0.3
    },
    "gmsc": {
      "auc": 0.77113,
      "pr_auc": 0.24962,
      "brier": 0.08841,
      "ece": 0.18062,
      "gini": 0.54226,
      "n": 150000,
      "base_rate": 0.06684
    },
    "home_credit": {
      "auc": 0.57854,
      "pr_auc": 0.10119,
      "brier": 0.11983,
      "ece": 0.18925,
      "gini": 0.15707,
      "n": 300000,
      "base_rate": 0.08076
    },
    "lending_club": {
      "auc": 0.53986,
      "pr_auc": 0.23571,
      "brier": 0.18794,
      "ece": 0.14079,
      "gini": 0.07971,
      "n": 300000,
      "base_rate": 0.21212
    },
    "sba": {
      "auc": 0.25738,
      "pr_auc": 0.11727,
      "brier": 0.16272,
      "ece": 0.15218,
      "gini": -0.48525,
      "n": 300000,
      "base_rate": 0.17512
    },
    "taiwan": {
      "auc": 0.69894,
      "pr_auc": 0.40759,
      "brier": 0.17723,
      "ece": 0.14459,
      "gini": 0.39788,
      "n": 30000,
      "base_rate": 0.2212
    }
  },
  "notes": {
    "berka_leakage_fix": "Berka aggregates ONLY pre-origination transactions. Using the full history leaks the label (71% of transactions post-date the loan) and yields AUC 1.0000, which is not reported."
  },
  "pooled_raw": {
    "uncalibrated_test": {
      "auc": 0.8594,
      "pr_auc": 0.58403,
      "brier": 0.15075,
      "ece": 0.21275,
      "gini": 0.7188,
      "n": 270421,
      "base_rate": 0.14555
    },
    "calibrated_test": {
      "auc": 0.85932,
      "pr_auc": 0.57674,
      "brier": 0.08721,
      "ece": 0.00149,
      "gini": 0.71864,
      "n": 270421,
      "base_rate": 0.14555
    },
    "per_member_test": {
      "lgbm": {
        "auc": 0.86009,
        "pr_auc": 0.58794,
        "brier": 0.14937,
        "ece": 0.20785,
        "gini": 0.72019,
        "n": 270421,
        "base_rate": 0.14555
      },
      "xgb": {
        "auc": 0.85282,
        "pr_auc": 0.56994,
        "brier": 0.15216,
        "ece": 0.21138,
        "gini": 0.70564,
        "n": 270421,
        "base_rate": 0.14555
      },
      "cat": {
        "auc": 0.85869,
        "pr_auc": 0.58063,
        "brier": 0.15401,
        "ece": 0.21902,
        "gini": 0.71738,
        "n": 270421,
        "base_rate": 0.14555
      }
    },
    "split": {
      "fit": 649009,
      "cal": 162252,
      "test": 270421
    },
    "train_seconds": 134.0,
    "members": [
      "lgbm",
      "xgb",
      "cat"
    ],
    "datasets": [
      "sba",
      "lending_club",
      "home_credit",
      "gmsc",
      "taiwan",
      "german",
      "berka"
    ]
  },
  "lodo_raw": {
    "berka": {
      "auc": 0.46123,
      "pr_auc": 0.10863,
      "brier": 0.10266,
      "ece": 0.04246,
      "gini": -0.07754,
      "n": 682,
      "base_rate": 0.11144
    },
    "german": {
      "auc": 0.46458,
      "pr_auc": 0.29047,
      "brier": 0.23741,
      "ece": 0.15139,
      "gini": -0.07084,
      "n": 1000,
      "base_rate": 0.3
    },
    "gmsc": {
      "auc": 0.77861,
      "pr_auc": 0.28425,
      "brier": 0.08594,
      "ece": 0.17679,
      "gini": 0.55722,
      "n": 150000,
      "base_rate": 0.06684
    },
    "home_credit": {
      "auc": 0.4991,
      "pr_auc": 0.07872,
      "brier": 0.22521,
      "ece": 0.37382,
      "gini": -0.0018,
      "n": 300000,
      "base_rate": 0.08076
    },
    "lending_club": {
      "auc": 0.52276,
      "pr_auc": 0.22327,
      "brier": 0.18042,
      "ece": 0.11233,
      "gini": 0.04551,
      "n": 300000,
      "base_rate": 0.21212
    },
    "sba": {
      "auc": 0.41031,
      "pr_auc": 0.15392,
      "brier": 0.17302,
      "ece": 0.1638,
      "gini": -0.17938,
      "n": 300000,
      "base_rate": 0.17512
    },
    "taiwan": {
      "auc": 0.7038,
      "pr_auc": 0.45141,
      "brier": 0.15338,
      "ece": 0.07019,
      "gini": 0.40759,
      "n": 30000,
      "base_rate": 0.2212
    }
  },
  "lodo_mean_auc": 0.5377,
  "lodo_raw_mean_auc": 0.5486
}