Upload src/models/ensemble.py with huggingface_hub

src/models/ensemble.py

@@ -0,0 +1,140 @@
+# src/models/ensemble.py
+#
+# Multi-seed OOF ensemble.
+# 3 seeds × 3 model types × 5 folds = 45 GBM runs.
+# Models: LGBM-RMSE, CatBoost, XGBoost.
+# All trained on z-scored targets, predictions inverse-transformed.
+
+import numpy as np
+import lightgbm as lgb
+import catboost as cb
+import xgboost as xgb
+import joblib
+from pathlib import Path
+from scipy.stats import pearsonr
+from sklearn.model_selection import KFold
+
+
+class TargetScaler:
+    """Z-score normalization on training targets only."""
+    def fit(self, y):
+        self.mu  = y.mean()
+        self.std = y.std()
+        return self
+    def transform(self, y):   return (y - self.mu) / self.std
+    def inverse(self, y):     return y * self.std + self.mu
+
+
+def _lgbm_rmse(seed, lr, n_trees):
+    return lgb.LGBMRegressor(
+        objective='regression', num_leaves=63, max_depth=7,
+        learning_rate=lr, n_estimators=n_trees,
+        min_child_samples=25, subsample=0.75, colsample_bytree=0.75,
+        reg_alpha=0.2, reg_lambda=2.0,
+        random_state=seed, n_jobs=4, verbose=-1,
+    )
+
+def _catboost(seed, lr, n_trees, early_stop):
+    return cb.CatBoostRegressor(
+        depth=7, learning_rate=lr, iterations=n_trees,
+        l2_leaf_reg=5.0, subsample=0.75, min_data_in_leaf=25,
+        loss_function='RMSE', eval_metric='RMSE',
+        early_stopping_rounds=early_stop,
+        random_seed=seed, verbose=0, task_type='CPU',
+    )
+
+def _xgboost(seed, lr, n_trees, early_stop):
+    return xgb.XGBRegressor(
+        n_estimators=n_trees, max_depth=6, learning_rate=lr,
+        subsample=0.75, colsample_bytree=0.75,
+        reg_alpha=0.2, reg_lambda=2.0, min_child_weight=6,
+        early_stopping_rounds=early_stop, eval_metric='rmse',
+        random_state=seed, n_jobs=4, verbosity=0,
+    )
+
+
+def _fit(model, Xtr, ytr, Xval, yval, early_stop):
+    if isinstance(model, lgb.LGBMRegressor):
+        model.fit(Xtr, ytr, eval_set=[(Xval, yval)],
+                  callbacks=[lgb.early_stopping(early_stop, verbose=False),
+                              lgb.log_evaluation(-1)])
+    elif isinstance(model, cb.CatBoostRegressor):
+        model.fit(Xtr, ytr, eval_set=(Xval, yval), use_best_model=True)
+    else:
+        model.fit(Xtr, ytr, eval_set=[(Xval, yval)], verbose=False)
+    return model
+
+
+def run_oof(X_train, y_train_raw, X_test,
+            seeds, n_folds, lr, n_trees, early_stop,
+            models_dir: Path = None) -> tuple:
+    """
+    Multi-seed OOF stacking.
+    3 model types: LGBM, CatBoost, XGBoost.
+
+    Args:
+        models_dir: if provided, saves each fold model as
+                    fold_model_s{seed}_{type}_f{fold}.pkl
+                    Required for CASF-2013 zero-shot evaluation.
+
+    Returns:
+        oof_matrix   [N_train, n_seeds * 3]
+        test_matrix  [N_test,  n_seeds * 3]
+        scaler       fitted TargetScaler
+    """
+    scaler  = TargetScaler().fit(y_train_raw)
+    y_train = scaler.transform(y_train_raw)
+    kf      = KFold(n_splits=n_folds, shuffle=True, random_state=42)
+
+    n_cols   = len(seeds) * 3
+    oof_mat  = np.zeros((len(X_train), n_cols))
+    test_mat = np.zeros((len(X_test),  n_cols))
+
+    for si, seed in enumerate(seeds):
+        print(f"\n  Seed {seed}  ({si+1}/{len(seeds)})")
+
+        oof_lr = np.zeros(len(X_train))
+        oof_cb = np.zeros(len(X_train))
+        oof_xb = np.zeros(len(X_train))
+
+        t_lr = np.zeros((len(X_test), n_folds))
+        t_cb = np.zeros((len(X_test), n_folds))
+        t_xb = np.zeros((len(X_test), n_folds))
+
+        for fold, (tri, vali) in enumerate(kf.split(X_train)):
+            Xtr, Xval = X_train[tri], X_train[vali]
+            ytr, yval = y_train[tri], y_train[vali]
+
+            mlr = _fit(_lgbm_rmse(seed, lr, n_trees),           Xtr, ytr, Xval, yval, early_stop)
+            mcb = _fit(_catboost(seed, lr, n_trees, early_stop), Xtr, ytr, Xval, yval, early_stop)
+            mxb = _fit(_xgboost(seed, lr, n_trees, early_stop), Xtr, ytr, Xval, yval, early_stop)
+
+            # Save fold models for zero-shot evaluation on new test sets
+            if models_dir is not None:
+                models_dir = Path(models_dir)
+                models_dir.mkdir(exist_ok=True)
+                joblib.dump(mlr, models_dir / f"fold_model_s{seed}_lgbm_f{fold}.pkl")
+                joblib.dump(mcb, models_dir / f"fold_model_s{seed}_cb_f{fold}.pkl")
+                joblib.dump(mxb, models_dir / f"fold_model_s{seed}_xgb_f{fold}.pkl")
+
+            oof_lr[vali] = mlr.predict(Xval)
+            oof_cb[vali] = mcb.predict(Xval)
+            oof_xb[vali] = mxb.predict(Xval)
+
+            t_lr[:, fold] = mlr.predict(X_test)
+            t_cb[:, fold] = mcb.predict(X_test)
+            t_xb[:, fold] = mxb.predict(X_test)
+
+        base = si * 3
+        oof_mat[:, base+0] = scaler.inverse(oof_lr)
+        oof_mat[:, base+1] = scaler.inverse(oof_cb)
+        oof_mat[:, base+2] = scaler.inverse(oof_xb)
+
+        test_mat[:, base+0] = scaler.inverse(t_lr.mean(1))
+        test_mat[:, base+1] = scaler.inverse(t_cb.mean(1))
+        test_mat[:, base+2] = scaler.inverse(t_xb.mean(1))
+
+        p = pearsonr(oof_mat[:, base:base+3].mean(1), y_train_raw)[0]
+        print(f"    OOF Pearson (seed {seed}): {p:.4f}")
+
+    return oof_mat, test_mat, scaler