Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V132.4 - GEM-Architect: Deep Diagnostic Mode)
+# 🧪 backtest_engine.py (V132.5 - GEM-Architect: Full Diagnostics)
 # ============================================================
 
 import asyncio
@@ -103,102 +103,68 @@ def _transform_window_for_pattern(df_window):
     except: return None
 
 def calculate_sniper_features_exact(df):
-    """
-    Sniper Features Calculation - TRUE UNIVERSAL MODE (Z-SCORE).
-    Converts structural features to Z-Scores to bypass scale issues.
-    """
-    # 1. Standard Features
     d = df.copy()
     c = d['close']; h = d['high']; l = d['low']; v = d['volume']; o = d['open']
     
-    # --- Helper: Rolling Z-Score ---
-    def _z_roll(x, w=200): # Window 200 is standard for regime detection
+    def _z_roll(x, w=200):
         r = x.rolling(w).mean()
         s = x.rolling(w).std().replace(0, np.nan)
         return ((x - r) / s).fillna(0)
 
-    # Basic Returns (Keep as percentages, trees handle these well)
     d['return_1m'] = c.pct_change(1).fillna(0)
     d['return_3m'] = c.pct_change(3).fillna(0)
     d['return_5m'] = c.pct_change(5).fillna(0)
     d['return_15m'] = c.pct_change(15).fillna(0)
 
-    # Technicals (RSI is bounded 0-100, usually safe)
     d['rsi_14'] = ta.rsi(c, length=14).fillna(50)
-    
     ema_9 = ta.ema(c, length=9).fillna(c)
     ema_21 = ta.ema(c, length=21).fillna(c)
-    
-    # Slopes/Distances -> Normalized
     d['ema_9_slope'] = ((ema_9 - ema_9.shift(1)) / ema_9.shift(1).replace(0, np.nan)).fillna(0)
     d['ema_21_dist'] = ((c - ema_21) / ema_21.replace(0, np.nan)).fillna(0)
     
-    # --- TRANSFORM 1: ATR (Vol) -> Z-Score ---
-    # Instead of raw value or %, check if Volatility is spiking relative to history
     atr_raw = ta.atr(h, l, c, length=100).fillna(0)
     d['atr'] = _z_roll(atr_raw, 500) 
-    
-    # Volume Z-Score
     d['vol_zscore_50'] = _z_roll(v, 50)
-    
-    # Candle Geometry
     rng = (h - l).replace(0, 1e-9)
-    d['candle_range'] = _z_roll(rng, 500) # Is this candle huge relative to history?
+    d['candle_range'] = _z_roll(rng, 500)
     d['close_pos_in_range'] = ((c - l) / rng).fillna(0.5)
 
-    # --- TRANSFORM 2: Liquidity Proxies -> Z-Score ---
-    # This fixes the Amihud 1e-8 issue completely.
-    
-    # Amihud
     d['dollar_vol'] = c * v
     amihud_raw = (d['return_1m'].abs() / d['dollar_vol'].replace(0, np.nan)).fillna(0)
     d['amihud'] = _z_roll(amihud_raw, 500)
     
-    # Roll Spread
     dp = c.diff()
     roll_cov = dp.rolling(64).cov(dp.shift(1)).fillna(0)
     roll_spread_raw = (2 * np.sqrt(np.maximum(0, -roll_cov)))
     d['roll_spread'] = _z_roll(roll_spread_raw, 500)
     
-    # OFI (Order Flow)
     sign = np.sign(c.diff()).fillna(0)
     d['signed_vol'] = sign * v
     ofi_raw = d['signed_vol'].rolling(30).sum().fillna(0)
     d['ofi'] = _z_roll(ofi_raw, 500)
     
-    # VPIN
     buy_vol = (sign > 0) * v
     sell_vol = (sign < 0) * v
     imb = (buy_vol.rolling(60).sum() - sell_vol.rolling(60).sum()).abs()
     tot = v.rolling(60).sum().replace(0, np.nan)
-    d['vpin'] = (imb / tot).fillna(0) # VPIN is a ratio (0-1), usually safe without Z-score, but let's keep it raw
+    d['vpin'] = (imb / tot).fillna(0)
     
-    # Garman-Klass Volatility
     rv_gk_raw = ((np.log(h / l)**2) / 2) - ((2 * np.log(2) - 1) * (np.log(c / o)**2))
     d['rv_gk'] = _z_roll(rv_gk_raw.fillna(0), 500)
     
-    # VWAP Deviation
     vwap_win = 20
     vwap = (d['dollar_vol'].rolling(vwap_win).sum() / v.rolling(vwap_win).sum().replace(0, np.nan)).fillna(c)
     d['vwap_dev'] = _z_roll((c - vwap), 500)
     
-    # Liquidity Score (Composite - Already using internal Z-scores in logic, but let's re-calc)
-    # Note: We use the already Z-scored columns now where possible
     d['L_score'] = (
-        d['vol_zscore_50'] + 
-        (-d['amihud']) +  # Low illiquidity is good
-        (-d['roll_spread']) + 
-        (-d['rv_gk'].abs()) + 
-        (-d['vwap_dev'].abs()) + 
-        d['ofi']
+        d['vol_zscore_50'] + (-d['amihud']) + (-d['roll_spread']) + 
+        (-d['rv_gk'].abs()) + (-d['vwap_dev'].abs()) + d['ofi']
     ).fillna(0)
     
     return sanitize_features(d)
 
 def calculate_titan_features_real(df):
-    """Titan features with strict Infinity handling."""
     df = df.copy()
-    
     df['RSI'] = ta.rsi(df['close'], 14)
     macd = ta.macd(df['close'])
     if macd is not None: 
@@ -269,15 +235,12 @@ def calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m):
         X_df = pd.concat(parts, axis=1)
         return sanitize_features(X_df).values 
     except Exception as e:
-        print(f"Legacy V2 Vec Error: {e}")
         return None
 
 def calculate_legacy_v3_vectorized(df_1m, df_5m, df_15m):
-    """Legacy V3 Safe Calc."""
     try:
         def calc_v3_base(df, prefix=""):
             d = df.copy()
-            # Initialize All
             targets = ['rsi', 'rsi_slope', 'macd_h', 'macd_h_slope', 'adx', 'dmp', 'dmn', 
                        'trend_net_force', 'ema_20', 'ema_50', 'ema_200', 'dist_ema20', 
                        'dist_ema50', 'dist_ema200', 'slope_ema50', 'atr', 'atr_rel', 
@@ -398,7 +361,6 @@ class HeavyDutyBacktester:
                 return raw 
             return model.predict(X)
         except Exception as e:
-            # print(f"⚠️ {model_name} Error: {e}") 
             return np.zeros(len(X) if hasattr(X, '__len__') else 0)
 
     def _extract_probs(self, raw_preds):
@@ -450,10 +412,8 @@ class HeavyDutyBacktester:
         df_5m = df_1m.resample('5T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
         df_15m = df_1m.resample('15T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
 
-        # 1. Sniper (True Match Logic)
+        # 1. Sniper
         df_sniper_feats = calculate_sniper_features_exact(df_1m)
-        
-        # Calculate L1 Score manually (since new Sniper function returns pure features)
         rel_vol = df_1m['volume'] / (df_1m['volume'].rolling(50).mean() + 1e-9)
         l1_score = (rel_vol * 10) + ((ta.atr(df_1m['high'], df_1m['low'], df_1m['close'], 14)/df_1m['close']) * 1000)
         
@@ -497,72 +457,36 @@ class HeavyDutyBacktester:
                 res_titan = self.proc.titan.model.predict(xgb.DMatrix(X_titan_df.values, feature_names=feats))
             except Exception as e: print(f"Titan Error: {e}")
 
-        # 4. Sniper (GEM-Architect Refactor: Deep Diagnostic Mode)
+        # 4. Sniper
         res_sniper = np.full(len(df_candidates), 0.5)
         sniper_instance = self.proc.sniper
-        
         if sniper_instance and getattr(sniper_instance, 'models', []):
             try:
-                # A. Identify Required Features
                 required_features = getattr(sniper_instance, 'feature_names', [])
                 if not required_features and hasattr(sniper_instance.models[0], 'feature_name'):
                      required_features = sniper_instance.models[0].feature_name()
 
-                # B. Build Maps
                 source_cols = df_sniper_feats.columns
                 def normalize_name(s): return s.lower().replace('_', '').replace('-', '').replace(' ', '')
                 col_map_lower = {col.lower(): col for col in source_cols}
                 col_map_fuzzy = {normalize_name(col): col for col in source_cols}
                 
-                # C. Construct X_final
                 X_final = pd.DataFrame(index=df_candidates.index)
-                missing_feats = []
-                found_feats = []
-
                 if required_features:
                     for req_feat in required_features:
                         req_norm = normalize_name(req_feat)
-                        if req_feat in source_cols:
-                            X_final[req_feat] = df_sniper_feats.loc[df_candidates.index, req_feat]
-                            found_feats.append(f"{req_feat}(Direct)")
-                        elif req_feat.lower() in col_map_lower:
-                            real_col = col_map_lower[req_feat.lower()]
-                            X_final[req_feat] = df_sniper_feats.loc[df_candidates.index, real_col]
-                            found_feats.append(f"{req_feat}->{real_col}")
-                        elif req_norm in col_map_fuzzy:
-                            real_col = col_map_fuzzy[req_norm]
-                            X_final[req_feat] = df_sniper_feats.loc[df_candidates.index, real_col]
-                            found_feats.append(f"{req_feat}~>{real_col}")
-                        else:
-                            missing_feats.append(req_feat)
-                            X_final[req_feat] = 0.0
-                else:
-                    X_final = df_sniper_feats.loc[df_candidates.index]
-
-                # D. Deep Diagnostics
-                print(f"     🕵️ [Sniper Diag] Expected {len(required_features)} | Found {len(required_features) - len(missing_feats)} | Missing {len(missing_feats)}")
-                if missing_feats:
-                     print(f"     ❌ MISSING: {missing_feats}")
-                if found_feats:
-                     print(f"     ✅ MAPPED (First 5): {found_feats[:5]}")
-                
-                # E. Data Stats Check
-                feat_stats = X_final.describe().T[['mean', 'min', 'max']]
-                print(f"     📈 [Sniper Input Stats] (First 3 Feats):\n{feat_stats.head(3)}")
+                        if req_feat in source_cols: X_final[req_feat] = df_sniper_feats.loc[df_candidates.index, req_feat]
+                        elif req_feat.lower() in col_map_lower: X_final[req_feat] = df_sniper_feats.loc[df_candidates.index, col_map_lower[req_feat.lower()]]
+                        elif req_norm in col_map_fuzzy: X_final[req_feat] = df_sniper_feats.loc[df_candidates.index, col_map_fuzzy[req_norm]]
+                        else: X_final[req_feat] = 0.0
+                else: X_final = df_sniper_feats.loc[df_candidates.index]
 
-                # F. Predict
                 preds = []
                 for m in sniper_instance.models:
                     X_in = X_final.astype(np.float32)
                     raw_p = self._extract_probs(self._smart_predict(m, X_in, "Sniper"))
                     preds.append(raw_p)
-                
                 res_sniper = np.mean(preds, axis=0)
-                
-                p_min, p_max, p_mean = res_sniper.min(), res_sniper.max(), res_sniper.mean()
-                n_high = np.sum(res_sniper > 0.6)
-                print(f"     🎯 [Sniper Output] Min:{p_min:.4f} Max:{p_max:.4f} Mean:{p_mean:.4f} | Signals(>0.6): {n_high}")
-
             except Exception as e: 
                 print(f"❌ Sniper Inference Error: {e}")
                 traceback.print_exc()
@@ -585,10 +509,10 @@ class HeavyDutyBacktester:
         hydra_models = getattr(self.proc.guardian_hydra, 'models', {})
         if hydra_models and 'crash' in hydra_models:
             try:
-                df_sniper_base = df_1m.copy() # Reuse basic DF
+                df_sniper_base = df_1m.copy() 
                 df_sniper_base['rsi_14'] = ta.rsi(df_sniper_base['close'], 14).fillna(50)
                 df_sniper_base['atr'] = ta.atr(df_sniper_base['high'], df_sniper_base['low'], df_sniper_base['close'], 14).fillna(0)
-                df_sniper_base['rel_vol'] = rel_vol # Reused
+                df_sniper_base['rel_vol'] = rel_vol 
 
                 global_hydra_feats = np.column_stack([
                     df_sniper_base['rsi_14'], df_sniper_base['rsi_14'], df_sniper_base['rsi_14'],
@@ -728,10 +652,33 @@ class HeavyDutyBacktester:
             
             final_bal = bal + alloc
             profit = final_bal - initial_capital
+            
+            # --- Detailed Stats Calculation ---
             tot = len(log)
-            wins = sum(1 for x in log if x['pnl'] > 0)
-            win_rate = (wins/tot*100) if tot else 0
+            winning_trades = [x for x in log if x['pnl'] > 0]
+            losing_trades = [x for x in log if x['pnl'] <= 0]
             
+            win_count = len(winning_trades)
+            loss_count = len(losing_trades)
+            win_rate = (win_count/tot*100) if tot else 0
+            
+            avg_win = np.mean([x['pnl'] for x in winning_trades]) if winning_trades else 0
+            avg_loss = np.mean([x['pnl'] for x in losing_trades]) if losing_trades else 0
+            
+            gross_profit = sum([x['pnl'] for x in winning_trades])
+            gross_loss = abs(sum([x['pnl'] for x in losing_trades]))
+            profit_factor = (gross_profit / gross_loss) if gross_loss > 0 else 99.9
+
+            # Streaks
+            max_win_s = 0; max_loss_s = 0; curr_w = 0; curr_l = 0
+            for t in log:
+                if t['pnl'] > 0:
+                    curr_w += 1; curr_l = 0
+                    if curr_w > max_win_s: max_win_s = curr_w
+                else:
+                    curr_l += 1; curr_w = 0
+                    if curr_l > max_loss_s: max_loss_s = curr_l
+
             cons_trades = [x for x in log if x['consensus']]
             n_cons = len(cons_trades)
             agree_rate = (n_cons/tot*100) if tot else 0
@@ -741,6 +688,10 @@ class HeavyDutyBacktester:
             res.append({
                 'config': cfg, 'final_balance': final_bal, 'net_profit': profit, 
                 'total_trades': tot, 'win_rate': win_rate, 'max_drawdown': 0,
+                'win_count': win_count, 'loss_count': loss_count,
+                'avg_win': avg_win, 'avg_loss': avg_loss,
+                'max_win_streak': max_win_s, 'max_loss_streak': max_loss_s,
+                'profit_factor': profit_factor,
                 'consensus_agreement_rate': agree_rate,
                 'high_consensus_win_rate': cons_win_rate,
                 'high_consensus_avg_pnl': cons_avg_pnl
@@ -766,6 +717,18 @@ class HeavyDutyBacktester:
         results_list.sort(key=lambda x: x['net_profit'], reverse=True)
         best = results_list[0]
 
+        # --- AUTO-DIAGNOSIS LOGIC ---
+        diag = []
+        if best['total_trades'] > 2000 and best['net_profit'] < 10:
+            diag.append("⚠️ Overtrading: Too many trades for low profit.")
+        if best['win_rate'] > 55 and best['net_profit'] < 0:
+            diag.append("⚠️ Fee Burn: High win rate but fees are eating profits.")
+        if abs(best['avg_loss']) > best['avg_win']:
+            diag.append("⚠️ Risk/Reward Inversion: Avg Loss > Avg Win.")
+        if best['max_loss_streak'] > 10:
+            diag.append("⚠️ Consecutive Loss Risk: Strategy prone to streak failures.")
+        if not diag: diag.append("✅ System Healthy.")
+
         print("\n" + "="*60)
         print(f"🏆 CHAMPION REPORT [{target_regime}]:")
         print(f"   💰 Final Balance:   ${best['final_balance']:,.2f}")
@@ -773,12 +736,16 @@ class HeavyDutyBacktester:
         print("-" * 60)
         print(f"   📊 Total Trades:    {best['total_trades']}")
         print(f"   📈 Win Rate:        {best['win_rate']:.1f}%")
+        print(f"   ✅ Winning Trades:  {best['win_count']} (Avg: {best['avg_win']*100:.2f}%)")
+        print(f"   ❌ Losing Trades:   {best['loss_count']} (Avg: {best['avg_loss']*100:.2f}%)")
+        print(f"   🌊 Max Streaks:     Win {best['max_win_streak']} | Loss {best['max_loss_streak']}")
+        print(f"   ⚖️ Profit Factor:   {best['profit_factor']:.2f}")
         print("-" * 60)
         print(f"   🧠 CONSENSUS ANALYTICS:")
         print(f"   🤝 Model Agreement Rate:     {best['consensus_agreement_rate']:.1f}%")
         print(f"   🌟 High-Consensus Win Rate:  {best['high_consensus_win_rate']:.1f}%")
-        print(f"   💎 High-Consensus Avg PnL:   {best['high_consensus_avg_pnl']:.2f}%")
         print("-" * 60)
+        print(f"   🩺 DIAGNOSIS: {' '.join(diag)}")
         print(f"   ⚙️ Oracle={best['config']['oracle_thresh']:.2f} | Sniper={best['config']['sniper_thresh']:.2f} | Hydra={best['config']['hydra_thresh']:.2f}")
         print("="*60)
         return best['config'], best