Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V113.0 - GEM-Architect: Global Pre-Inference)
+# 🧪 backtest_engine.py (V115.0 - GEM-Architect: Full Grid Density)
 # ============================================================
 
 import asyncio
@@ -23,7 +23,6 @@ try:
     from r2 import R2Service
     import ccxt.async_support as ccxt 
     import xgboost as xgb 
-    import lightgbm as lgb 
 except ImportError:
     pass
 
@@ -34,28 +33,26 @@ class HeavyDutyBacktester:
     def __init__(self, data_manager, processor):
         self.dm = data_manager
         self.proc = processor
-        self.GRID_DENSITY = 5 
+        
+        # 🎛️ GRID DENSITY CONTROL
+        # 3 = Low (243 Scenarios) - Fast
+        # 4 = Med (1024 Scenarios) - Balanced
+        # 5 = High (3125 Scenarios) - Deep Search
+        self.GRID_DENSITY = 3  
+        
         self.INITIAL_CAPITAL = 10.0
         self.TRADING_FEES = 0.001 
         self.MAX_SLOTS = 4         
         
         self.TARGET_COINS = [
-            'SOL/USDT', 'XRP/USDT', 'DOGE/USDT', 'ADA/USDT', 'AVAX/USDT', 'LINK/USDT', 
-            'TON/USDT', 'INJ/USDT', 'APT/USDT', 'OP/USDT', 'ARB/USDT', 'SUI/USDT', 
-            'SEI/USDT', 'TIA/USDT', 'ETH/USDT', 'NEAR/USDT', 'RUNE/USDT', 'PYTH/USDT', 
-            'WIF/USDT', 'PEPE/USDT', 'SHIB/USDT', 'TRX/USDT', 'DOT/USDT', 'UNI/USDT', 
-            'ONDO/USDT', 'ENA/USDT', 'HBAR/USDT', 'XLM/USDT', 'TAO/USDT', 'BTC/USDT', 
-            'ZRO/USDT', 'KCS/USDT', 'ICP/USDT', 'SAND/USDT', 'AXS/USDT', 'APE/USDT', 
-            'GMT/USDT', 'CHZ/USDT', 'CFX/USDT', 'LDO/USDT', 'FET/USDT', 'JTO/USDT', 
-            'STRK/USDT', 'BLUR/USDT', 'ALT/USDT', 'JUP/USDT', 'PENDLE/USDT', 'ETHFI/USDT', 
-            'MEME/USDT', 'ATOM/USDT'
+            'SOL/USDT', 'XRP/USDT', 'DOGE/USDT'
         ]
         
         self.force_start_date = None 
         self.force_end_date = None   
         
         if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        print(f"🧪 [Backtest V113.0] Pre-Inference Velocity Mode (Target: 60s).")
+        print(f"🧪 [Backtest V115.0] Grid Density: {self.GRID_DENSITY} | Full Stack Optimization.")
 
     def set_date_range(self, start_str, end_str):
         self.force_start_date = start_str
@@ -237,7 +234,6 @@ class HeavyDutyBacktester:
         legacy_v2 = getattr(self.proc.guardian_legacy, 'model_v2', None)
         
         # 5. 🔥 PRE-CALCULATE LEGACY V2 (GLOBAL) 🔥
-        # V2 depends only on structure, not entry price. We can predict for ALL rows at once.
         global_v2_probs = np.zeros(len(fast_1m['close']))
         
         if legacy_v2:
@@ -284,7 +280,6 @@ class HeavyDutyBacktester:
             except Exception as e: print(f"V2 Error: {e}")
 
         # 6. 🔥 PRE-ASSEMBLE HYDRA STATIC (GLOBAL) 🔥
-        # Hydra needs PnL (dynamic), but 90% features are static.
         global_hydra_static = None
         if hydra_models:
             print(f"     🚀 Pre-assembling Hydra features...", flush=True)
@@ -298,8 +293,6 @@ class HeavyDutyBacktester:
                 h_atr = fast_1m['atr']
                 h_close = fast_1m['close']
                 
-                # We store these separate to combine inside loop efficiently
-                # [rsi1, rsi5, rsi15, bb, vol, atr, close]
                 global_hydra_static = np.column_stack([h_rsi_1m, h_rsi_5m, h_rsi_15m, h_bb, h_vol, h_atr, h_close])
             except: pass
 
@@ -386,7 +379,7 @@ class HeavyDutyBacktester:
             max_hydra_crash = 0.0; hydra_crash_time = 0
             if hydra_models and global_hydra_static is not None:
                 # Slice Static Feats
-                sl_static = global_hydra_static[start_idx:end_idx] # [rsi1, rsi5, rsi15, bb, vol, atr, close]
+                sl_static = global_hydra_static[start_idx:end_idx] 
                 
                 entry_price = fast_1m['close'][idx_1m]
                 sl_close = sl_static[:, 6]
@@ -405,13 +398,6 @@ class HeavyDutyBacktester:
                 
                 sl_atr_pct = sl_atr / sl_close
                 
-                # Map to Hydra Cols Order (Hardcoded for max speed)
-                # Cols: rsi_1m, rsi_5m, rsi_15m, bb_width, rel_vol, dist_ema20_1h, atr_pct, norm_pnl_r, max_pnl_r, dists..., time, entry, oracle, l2, target
-                
-                # Re-assemble only what is needed
-                # (Static 0-4) + (Zeros) + (Dynamic) + (Constants)
-                
-                # Create arrays for constants
                 zeros = np.zeros(240)
                 oracle_arr = np.full(240, oracle_conf)
                 l2_arr = np.full(240, 0.7)
@@ -437,7 +423,7 @@ class HeavyDutyBacktester:
 
             ai_results.append({
                 'timestamp': ts_val, 'symbol': sym, 'close': entry_price,
-                'real_titan': 0.6, 
+                'real_titan': 0.6, # Placeholder for real Titan score if available
                 'oracle_conf': oracle_conf, 
                 'sniper_score': sniper_score,
                 'risk_hydra_crash': max_hydra_crash,
@@ -459,7 +445,7 @@ class HeavyDutyBacktester:
         gc.collect()
 
     # ==============================================================
-    # PHASE 1 & 2 (Unchanged - Standard Optimization Logic)
+    # PHASE 1 & 2 (Enhanced with Consensus Analytics)
     # ==============================================================
     async def generate_truth_data(self):
         if self.force_start_date and self.force_end_date:
@@ -493,8 +479,14 @@ class HeavyDutyBacktester:
         
         for config in combinations_batch:
             wallet = { "balance": initial_capital, "allocated": 0.0, "positions": {}, "trades_history": [] }
-            w_titan = config['w_titan']; oracle_thresh = config.get('oracle_thresh', 0.6)
-            sniper_thresh = config.get('sniper_thresh', 0.4); hydra_thresh = config['hydra_thresh']
+            
+            # Param Extraction
+            oracle_thresh = config.get('oracle_thresh', 0.6)
+            sniper_thresh = config.get('sniper_thresh', 0.4)
+            hydra_thresh = config['hydra_thresh']
+            # Titan & Pattern weights are in config but not used for hard filtering here, 
+            # they are optimized for the DNA output.
+            
             peak_balance = initial_capital; max_drawdown = 0.0
 
             for ts, group in grouped_by_time:
@@ -511,8 +503,12 @@ class HeavyDutyBacktester:
                         if is_crash or pnl > 0.04 or pnl < -0.02:
                             wallet['balance'] += pos['size'] * (1 + pnl - (fees_pct*2))
                             wallet['allocated'] -= pos['size']
+                            # Add consensus data to history
+                            wallet['trades_history'].append({
+                                'pnl': pnl, 
+                                'consensus_score': pos['consensus_score']
+                            })
                             del wallet['positions'][sym]
-                            wallet['trades_history'].append({'pnl': pnl})
 
                 total_eq = wallet['balance'] + wallet['allocated']
                 if total_eq > peak_balance: peak_balance = total_eq
@@ -522,15 +518,22 @@ class HeavyDutyBacktester:
                 if len(wallet['positions']) < max_slots:
                     for _, row in group.iterrows():
                         if row['symbol'] in wallet['positions']: continue
+                        
+                        # Hard Filters
                         if row['oracle_conf'] < oracle_thresh: continue
                         if row['sniper_score'] < sniper_thresh: continue
                         
+                        # Consensus Calculation (Normalized)
+                        # Titan (default 0.6) + Oracle + Sniper
+                        cons_score = (row['real_titan'] + row['oracle_conf'] + row['sniper_score']) / 3.0
+
                         size = 10.0
                         if wallet['balance'] >= size:
                             wallet['positions'][row['symbol']] = {
                                 'entry': row['close'], 'size': size,
                                 'risk_hydra_crash': row['risk_hydra_crash'],
-                                'time_hydra_crash': row['time_hydra_crash']
+                                'time_hydra_crash': row['time_hydra_crash'],
+                                'consensus_score': cons_score
                             }
                             wallet['balance'] -= size
                             wallet['allocated'] += size
@@ -545,6 +548,7 @@ class HeavyDutyBacktester:
             max_win = max([t['pnl'] for t in trades]) if trades else 0
             max_loss = min([t['pnl'] for t in trades]) if trades else 0
             
+            # 1. Fix: Calculate Streaks
             max_win_streak = 0; max_loss_streak = 0; curr_w = 0; curr_l = 0
             for t in trades:
                 if t['pnl'] > 0:
@@ -554,32 +558,68 @@ class HeavyDutyBacktester:
                     curr_l += 1; curr_w = 0
                     if curr_l > max_loss_streak: max_loss_streak = curr_l
 
+            # 2. Fix: Consensus Analytics
+            high_cons_trades = [t for t in trades if t['consensus_score'] > 0.65]
+            low_cons_trades = [t for t in trades if t['consensus_score'] <= 0.65]
+            
+            hc_count = len(high_cons_trades)
+            hc_wins = len([t for t in high_cons_trades if t['pnl'] > 0])
+            hc_win_rate = (hc_wins/hc_count*100) if hc_count > 0 else 0
+            hc_avg_pnl = (sum([t['pnl'] for t in high_cons_trades]) / hc_count * 100) if hc_count > 0 else 0
+            
+            lc_count = len(low_cons_trades)
+            lc_wins = len([t for t in low_cons_trades if t['pnl'] > 0])
+            lc_win_rate = (lc_wins/lc_count*100) if lc_count > 0 else 0
+            
+            agreement_rate = (hc_count / total_t * 100) if total_t > 0 else 0.0
+
             results.append({
                 'config': config, 'final_balance': final_bal, 'net_profit': net_profit,
                 'total_trades': total_t, 'win_count': win_count, 'loss_count': loss_count,
                 'win_rate': win_rate, 'max_single_win': max_win, 'max_single_loss': max_loss,
-                'max_drawdown': max_drawdown * 100
+                'max_drawdown': max_drawdown * 100,
+                # New Fields
+                'max_win_streak': max_win_streak,
+                'max_loss_streak': max_loss_streak,
+                'consensus_agreement_rate': agreement_rate,
+                'high_consensus_win_rate': hc_win_rate,
+                'low_consensus_win_rate': lc_win_rate,
+                'high_consensus_avg_pnl': hc_avg_pnl
             })
             
         return results
 
     async def run_optimization(self, target_regime="RANGE"):
         await self.generate_truth_data()
-        oracle_range = [0.5, 0.6, 0.7]
-        sniper_range = [0.4, 0.5, 0.6]
-        hydra_range = [0.75, 0.85, 0.95]
+        
+        # 🔥 Dynamic Ranges based on GRID_DENSITY
+        density = self.GRID_DENSITY
+        
+        oracle_range = np.linspace(0.5, 0.8, density).tolist()
+        sniper_range = np.linspace(0.4, 0.7, density).tolist()
+        hydra_range = np.linspace(0.75, 0.95, density).tolist()
+        
+        # New Params (Titan & Pattern)
+        titan_range = np.linspace(0.4, 0.7, density).tolist()
+        pattern_range = np.linspace(0.2, 0.5, density).tolist()
+        
         combinations = []
-        for o, s, h in itertools.product(oracle_range, sniper_range, hydra_range):
+        # Full Stack Loop
+        for o, s, h, wt, wp in itertools.product(oracle_range, sniper_range, hydra_range, titan_range, pattern_range):
             combinations.append({
-                'w_titan': 0.5, 'w_struct': 0.3, 'thresh': 0.5, 
-                'oracle_thresh': o, 'sniper_thresh': s, 'hydra_thresh': h,
+                'w_titan': wt, 
+                'w_struct': wp, 
+                'thresh': 0.5, 
+                'oracle_thresh': o, 
+                'sniper_thresh': s, 
+                'hydra_thresh': h,
                 'legacy_thresh': 0.95
             })
             
         current_period_files = [os.path.join(CACHE_DIR, f) for f in os.listdir(CACHE_DIR) if f.endswith('_scores.pkl')]
         if not current_period_files: return None, None
         
-        print(f"\n🧩 [Phase 2] Optimizing {len(combinations)} Configs (Full Stack) for {target_regime}...")
+        print(f"\n🧩 [Phase 2] Optimizing {len(combinations)} Configs (Full Stack | Density {density}) for {target_regime}...")
         best_res = self._worker_optimize(combinations, current_period_files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS)
         if not best_res: return None, None
         best = sorted(best_res, key=lambda x: x['final_balance'], reverse=True)[0]
@@ -595,13 +635,19 @@ class HeavyDutyBacktester:
         print(f"   ❌ Losing Trades:   {best['loss_count']}")
         print(f"   📈 Win Rate:        {best['win_rate']:.1f}%")
         print("-" * 60)
+        print(f"   🧠 CONSENSUS ANALYTICS:")
+        print(f"   🤝 Model Agreement Rate:     {best['consensus_agreement_rate']:.1f}% (of all trades)")
+        print(f"   🌟 High-Consensus Win Rate:  {best['high_consensus_win_rate']:.1f}%")
+        print(f"   💎 High-Consensus Avg PnL:   {best['high_consensus_avg_pnl']:.2f}%")
+        print("-" * 60)
         print(f"   🟢 Max Single Win:  ${best['max_single_win']:.2f}")
         print(f"   🔴 Max Single Loss: ${best['max_single_loss']:.2f}")
         print(f"   🔥 Max Win Streak:  {best['max_win_streak']} trades")
         print(f"   🧊 Max Loss Streak: {best['max_loss_streak']} trades")
         print(f"   📉 Max Drawdown:    {best['max_drawdown']:.1f}%")
         print("-" * 60)
-        print(f"   ⚙️ Oracle={best['config']['oracle_thresh']} | Sniper={best['config']['sniper_thresh']} | Hydra={best['config']['hydra_thresh']}")
+        print(f"   ⚙️ Oracle={best['config']['oracle_thresh']:.2f} | Sniper={best['config']['sniper_thresh']:.2f} | Hydra={best['config']['hydra_thresh']:.2f}")
+        print(f"   ⚖️ Weights: Titan={best['config']['w_titan']:.2f} | Patterns={best['config']['w_struct']:.2f}")
         print("="*60)
         return best['config'], best
 
@@ -617,6 +663,9 @@ async def run_strategic_optimization_task():
         hub = AdaptiveHub(r2); await hub.initialize()
         optimizer = HeavyDutyBacktester(dm, proc)
         
+        # ⚡ ADJUST DENSITY HERE IF NEEDED
+        # optimizer.GRID_DENSITY = 3 (Default)
+        
         scenarios = [
             {"regime": "BULL", "start": "2024-01-01", "end": "2024-03-30"},
             {"regime": "BEAR", "start": "2023-08-01", "end": "2023-09-15"},