Classifier-family sweep: RF/ExtraTrees/MLP + rank-hybrid all rejected by CV

RandomForest (default/deep/class-balanced), ExtraTrees, and a small MLP all
lose 170-820 dev points to bagged logistic. RF ranks better at some origins
(AUC 0.94 vs 0.85 at s_23) but its compressed probabilities break topsum's
count; a rank_rf hybrid preserving logbag's probability scale while using
mean-rank ordering still loses (+92). Champion unchanged.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

EXPERIMENTS.md

@@ -133,6 +133,23 @@ untested); `logbag_aux*` variants await their feature columns in the table.
 **Conclusion**: every axis is flat or worse around the champion - remaining cost
 is structural (fast-plunge deaths), not hyperparameters.
 
+### Classifier-family addendum (RandomForest and friends)
+
+| model | dev | Δ | note |
+|---|---|---|---|
+| RandomForest (500, leaf 5) | 2273.1 | +199 | ranks well (AUC 0.94 at s_23 vs logbag 0.85) but probabilities too compressed for topsum's count |
+| RF deep (leaf 1, 1000) | 2325.9 | +252 | |
+| RF class-balanced | 2890.2 | +816 | worst config ever tested - inflated Σp42 over-includes massively |
+| ExtraTrees | 2245.3 | +171 | |
+| MLP (32,16, early stop) | 2768.7 | +695 | |
+| rank_rf hybrid (RF ranking, logbag scale - Σp42 preserved) | 2166.2 | +92 | RF's ranking edge does not transfer across origins even with the scale problem removed |
+
+Complete p42-classifier ledger: logistic-family (champion), HGB/GBM x4,
+Weibull AFT, discrete hazard (tied), RF x3, ExtraTrees, MLP, and five blends -
+linear + bagging wins throughout. With ~1000 positives, one dominant monotone
+signal (voltage level), and a planner that needs a stable probability scale,
+that is the expected outcome.
+
 ## Trajectory
 
 | champion | dev | test | combined (34) | all-48 metric | public LB |

batteryswap_example/models_cv.py

@@ -141,6 +141,31 @@ class SklearnRUL:
                                                     C=self.opts.get('C', 1.0)))
             self.clf = BaggingClassifier(
                 base, n_estimators=int(self.opts.get('bags', 15)), random_state=0)
+        elif self.kind == 'rf':
+            from sklearn.ensemble import RandomForestClassifier
+            cw = 'balanced_subsample' if self.opts.get('bal') else None
+            self.clf = make_pipeline(
+                SimpleImputer(strategy='median'),
+                RandomForestClassifier(
+                    n_estimators=int(self.opts.get('trees', 500)),
+                    min_samples_leaf=int(self.opts.get('leaf', 5)),
+                    max_features='sqrt', class_weight=cw,
+                    random_state=0, n_jobs=-1))
+        elif self.kind == 'xtrees':
+            from sklearn.ensemble import ExtraTreesClassifier
+            self.clf = make_pipeline(
+                SimpleImputer(strategy='median'),
+                ExtraTreesClassifier(n_estimators=500, min_samples_leaf=5,
+                                     max_features='sqrt', random_state=0,
+                                     n_jobs=-1))
+        elif self.kind == 'mlp':
+            from sklearn.neural_network import MLPClassifier
+            self.clf = make_pipeline(
+                SimpleImputer(strategy='median'),
+                StandardScaler(),
+                MLPClassifier(hidden_layer_sizes=(32, 16), alpha=1e-2,
+                              max_iter=1000, early_stopping=True,
+                              random_state=0))
         self.clf.fit(X, y)
 
         dying = train[train['dies42'] == 1]
@@ -158,6 +183,25 @@ class SklearnRUL:
         return out
 
 
+class RankBlendRF:
+    """logbag's probability scale with (logbag+RF) mean-rank ordering: keeps
+    sum(p42) exactly (topsum count unchanged) while letting the forest's ranking
+    reorder WHO is included - RF ranks better at some origins but its raw
+    probabilities are too compressed for count-tracking."""
+
+    def fit(self, train):
+        self.a = SklearnRUL('logbag').fit(train)
+        self.b = SklearnRUL('rf').fit(train)
+        return self
+
+    def predict(self, rows):
+        out = self.a.predict(rows)
+        mean_rank = (out['p42'].rank() + self.b.predict(rows)['p42'].rank()) / 2
+        vals = np.sort(out['p42'].values)                 # same multiset of values
+        out['p42'] = pd.Series(vals, index=mean_rank.sort_values().index).reindex(out.index)
+        return out
+
+
 class RankVoltage:
     """Control: score by v_last only; p42 is a rank in [0,1], day from DayQuantiles."""
 
@@ -230,11 +274,14 @@ class DiscreteHazard:
     BUCKET = 7.0
     K = 6  # buckets cover days (7k, 7(k+1)], k=0..5
 
+    def __init__(self, extra=None):
+        self.feats = FEATURES + (extra or [])
+
     def fit(self, train_censored):
         dur = train_censored['duration'].clip(upper=WINDOW_DAYS).values
         event = ((train_censored['event'] == 1) &
                  (train_censored['duration'] <= WINDOW_DAYS)).values
-        X = train_censored[FEATURES].values
+        X = train_censored[self.feats].values
         rows, ys, ks = [], [], []
         for k in range(self.K):
             lo, hi = self.BUCKET * k, self.BUCKET * (k + 1)
@@ -260,7 +307,7 @@ class DiscreteHazard:
         return self
 
     def predict(self, rows):
-        X = rows[FEATURES].values
+        X = rows[self.feats].values
         n = len(rows)
         haz = np.empty((n, self.K))
         for k in range(self.K):
@@ -334,6 +381,79 @@ class BlendDH:
         return out
 
 
+class LogBagUCB(LogBagFeats):
+    """Rank by mean + k*std across the bagged estimators: a battery the ensemble
+    disagrees about gets an inclusion boost (missing it costs ~4x including it).
+    N still comes from sum of mean p42, so the count stays calibrated."""
+
+    def __init__(self, extra, k=0.5):
+        super().__init__(extra)
+        self.k = k
+
+    def predict(self, rows):
+        out = self.days.predict(rows[FEATURES])
+        X = rows[self.feats]
+        Xi = pd.DataFrame(X, columns=self.feats)
+        probs = np.stack([est.predict_proba(Xi.iloc[:, feats])[:, 1]
+                          for est, feats in zip(self.clf.estimators_,
+                                                self.clf.estimators_features_)])
+        mean, std = probs.mean(axis=0), probs.std(axis=0)
+        out['p42'] = np.clip(mean + self.k * std, 0, 1)
+        out['p42_sum'] = mean.sum()
+        return out
+
+
+class LogBagLog(LogBagFeats):
+    """Same features, but log1p-compress the heavy-tailed ones so the linear
+    model is not dominated by extreme staleness/gap values."""
+
+    SKEWED = ['staleness', 'gap_max_60', 'n_30']
+
+    def _tx(self, rows):
+        X = rows[self.feats].copy()
+        for c in self.SKEWED:
+            if c in X:
+                X[c] = np.log1p(X[c].clip(lower=0))
+        return X
+
+    def fit(self, train):
+        from sklearn.ensemble import BaggingClassifier
+        X, y = self._tx(train), train['dies42']
+        base = make_pipeline(SimpleImputer(strategy='median'),
+                             StandardScaler(),
+                             LogisticRegression(max_iter=2000, C=1.0))
+        self.clf = BaggingClassifier(base, n_estimators=self.n_estimators,
+                                     random_state=0)
+        self.clf.fit(X, y)
+        dying = train[train['dies42'] == 1]
+        self.days = DayQuantiles().fit(dying[FEATURES], dying['days_to_eol'])
+        return self
+
+    def predict(self, rows):
+        out = self.days.predict(rows[FEATURES])
+        out['p42'] = self.clf.predict_proba(self._tx(rows))[:, 1]
+        return out
+
+
+class BlendDHAux:
+    """Mean of logbag_aux (closed windows) and DiscreteHazard-with-aux
+    (censored open windows): decorrelated training sets, shared feature set."""
+
+    def fit(self, train_censored):
+        # recover the scored origin T: censored durations end exactly at T
+        tc = train_censored
+        T = (tc['origin'] + pd.to_timedelta(tc['duration'], unit='D')).max()
+        closed = tc[tc['origin'] <= T - pd.Timedelta(days=WINDOW_DAYS)]
+        self.a = LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60']).fit(closed)
+        self.b = DiscreteHazard(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60']).fit(train_censored)
+        return self
+
+    def predict(self, rows):
+        out = self.a.predict(rows)
+        out['p42'] = 0.5 * (out['p42'] + self.b.predict(rows)['p42'])
+        return out
+
+
 class WeightedBagLogistic:
     """Bagged logistic with recency-weighted samples: pairs from recent origins
     count more, on the theory that fleet composition and degradation mix drift."""
@@ -417,8 +537,9 @@ class LogBagC:
 class LogBagFeats:
     """logbag with extra feature columns (aux usage-dynamics table)."""
 
-    def __init__(self, extra):
+    def __init__(self, extra, n_estimators=15):
         self.feats = FEATURES + extra
+        self.n_estimators = n_estimators
 
     def fit(self, train):
         from sklearn.ensemble import BaggingClassifier
@@ -426,7 +547,7 @@ class LogBagFeats:
         base = make_pipeline(SimpleImputer(strategy='median'),
                              StandardScaler(),
                              LogisticRegression(max_iter=2000, C=1.0))
-        self.clf = BaggingClassifier(base, n_estimators=15, random_state=0)
+        self.clf = BaggingClassifier(base, n_estimators=self.n_estimators, random_state=0)
         self.clf.fit(X, y)
         dying = train[train['dies42'] == 1]
         self.days = DayQuantiles().fit(dying[FEATURES], dying['days_to_eol'])
@@ -456,6 +577,26 @@ MODELS = {
     'blend_dh': BlendDH,
     'logbag_aux': lambda: LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60']),
     'logbag_aux2': lambda: LogBagFeats(['rng_30', 'rng_90', 'vstd_30']),
+    'rank_rf': RankBlendRF,
+    'logbag_proj': lambda: LogBagFeats(['v_proj']),
+    'logbag_rank': lambda: LogBagFeats(['r_vlast', 'r_tau']),
+    'logbag_rankproj': lambda: LogBagFeats(['v_proj', 'r_vlast', 'r_tau']),
+    'aux_rv': lambda: LogBagFeats(['rng_90', 'vstd_30']),
+    'aux_rvg': lambda: LogBagFeats(['rng_90', 'vstd_30', 'gap_max_60']),
+    'aux_rvr': lambda: LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio']),
+    'aux_all': lambda: LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60',
+                                    'hot_90', 'oob_90']),
+    'aux_proj': lambda: LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60',
+                                     'v_proj']),
+    'aux_rank': lambda: LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60',
+                                     'r_vlast', 'r_tau']),
+    'aux_bag30': lambda: LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60'], n_estimators=30),
+    'aux_bag50': lambda: LogBagFeats(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60'], n_estimators=50),
+    'dthaz_aux': lambda: DiscreteHazard(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60']),
+    'blend_dhaux': BlendDHAux,
+    'aux_ucb05': lambda: LogBagUCB(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60'], k=0.5),
+    'aux_ucb10': lambda: LogBagUCB(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60'], k=1.0),
+    'aux_log': lambda: LogBagLog(['rng_90', 'vstd_30', 'rng_ratio', 'gap_max_60']),
 }
 
 
@@ -809,7 +950,8 @@ def plan_tsroute(pred, included, locations, start_time, day_q,
            floor_day - defers uncertain insurance swaps that would otherwise
            pay early-replacement for weeks [floor_day 0 = off]
     """
-    n = int(round(plan_tsroute.alpha * pred['p42'].sum()))
+    psum = pred['p42_sum'].iloc[0] if 'p42_sum' in pred else pred['p42'].sum()
+    n = int(round(plan_tsroute.alpha * psum))
     chosen = pred['p42'].nlargest(n).index
     if plan_tsroute.p_be > 0:
         topups = pred.index[pred['p42'] >= plan_tsroute.p_be]
@@ -1029,7 +1171,7 @@ def walk_forward(table, scenario_ctx, origins, configs):
         for model_name in {c['model'] for c in configs}:
             if (model_name, idx) not in fitted:
                 factory = resolve_model(model_name)
-                if model_name in ('aft', 'dthaz', 'logbag_open', 'blend_dh'):
+                if model_name in ('aft', 'dthaz', 'dthaz_aux', 'logbag_open', 'blend_dh', 'blend_dhaux'):
                     train = censored_pairs(table, origin_time)
                 else:
                     train = training_pairs(table, origin_time)
@@ -1105,6 +1247,11 @@ def summarize(df, label):
 def main():
     dataset_path = Path(os.environ.get('BATTERYSWAP_DATASET_PATH', 'dataset'))
     table = load_features(dataset_path)
+    # derived: voltage projected through the reporting gap by the device's own
+    # slope, and leak-free within-origin fleet ranks (data <= origin only)
+    table['v_proj'] = table['v_last'] + table['slope_lastk'] * table['staleness']
+    table['r_vlast'] = table.groupby('origin_idx')['v_last'].rank(pct=True)
+    table['r_tau'] = table.groupby('origin_idx')['tau_projected'].rank(pct=True)
     try:
         from batteryswap_example.aux_features import load as load_aux
         table = table.merge(load_aux(dataset_path), on=['device_id', 'origin_idx'],