Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V110.1 - GEM-Architect: Fix idx_5m NameError)
+# 🧪 backtest_engine.py (V111.0 - GEM-Architect: Full Regime Loop Restored)
 # ============================================================
 
 import asyncio
@@ -55,7 +55,7 @@ class HeavyDutyBacktester:
         self.force_end_date = None   
         
         if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        print(f"🧪 [Backtest V110.1] Full Stack Simulation (Fixed idx_5m).")
+        print(f"🧪 [Backtest V111.0] Full Stack + Multi-Regime Strategic Loop.")
 
     def set_date_range(self, start_str, end_str):
         self.force_start_date = start_str
@@ -104,20 +104,23 @@ class HeavyDutyBacktester:
         return unique_candles
 
     # ==============================================================
-    # 🏎️ VECTORIZED INDICATORS
+    # 🏎️ VECTORIZED INDICATORS (ALL LAYERS)
     # ==============================================================
     def _calculate_indicators_vectorized(self, df, timeframe='1m'):
+        # 1. Basic Setup
         df['close'] = df['close'].astype(float)
         df['high'] = df['high'].astype(float)
         df['low'] = df['low'].astype(float)
         df['volume'] = df['volume'].astype(float)
         df['open'] = df['open'].astype(float)
         
+        # 2. Standard Indicators
         df['rsi'] = ta.rsi(df['close'], length=14)
         df['ema20'] = ta.ema(df['close'], length=20)
         df['ema50'] = ta.ema(df['close'], length=50)
         df['atr'] = ta.atr(df['high'], df['low'], df['close'], length=14)
         
+        # 3. Hydra
         if timeframe == '1m':
             sma20 = df['close'].rolling(20).mean()
             std20 = df['close'].rolling(20).std()
@@ -125,12 +128,14 @@ class HeavyDutyBacktester:
             df['vol_ma50'] = df['volume'].rolling(50).mean()
             df['rel_vol'] = df['volume'] / (df['vol_ma50'] + 1e-9)
 
+        # 4. Oracle
         df['slope'] = ta.slope(df['close'], length=7)
         vol_mean = df['volume'].rolling(20).mean()
         vol_std = df['volume'].rolling(20).std()
         df['vol_z'] = (df['volume'] - vol_mean) / (vol_std + 1e-9)
         df['atr_pct'] = df['atr'] / df['close']
 
+        # 5. Sniper (1m Only)
         if timeframe == '1m':
             df['ret'] = df['close'].pct_change()
             df['dollar_vol'] = df['close'] * df['volume']
@@ -163,19 +168,16 @@ class HeavyDutyBacktester:
             s = df['volume'].rolling(500).std()
             df['vol_zscore_50'] = ((df['volume'] - r) / s).fillna(0)
 
+        # 6. Legacy
         df['log_ret'] = np.log(df['close'] / df['close'].shift(1))
-        
         roll_max = df['high'].rolling(50).max()
         roll_min = df['low'].rolling(50).min()
         diff = (roll_max - roll_min).replace(0, 1e-9)
         df['fib_pos'] = (df['close'] - roll_min) / diff
-        
         df['trend_slope'] = (df['ema20'] - df['ema20'].shift(5)) / df['ema20'].shift(5)
         df['volatility'] = df['atr'] / df['close']
-        
         fib618 = roll_max - (diff * 0.382)
         df['dist_fib618'] = (df['close'] - fib618) / df['close']
-
         df['dist_ema50'] = (df['close'] - df['ema50']) / df['close']
         df['ema200'] = ta.ema(df['close'], length=200)
         df['dist_ema200'] = (df['close'] - df['ema200']) / df['close']
@@ -184,7 +186,7 @@ class HeavyDutyBacktester:
         return df
 
     # ==============================================================
-    # 🧠 CPU PROCESSING (Full Stack + Logs)
+    # 🧠 CPU PROCESSING (Full Stack Injection + Logs)
     # ==============================================================
     async def _process_data_in_memory(self, sym, candles, start_ms, end_ms):
         safe_sym = sym.replace('/', '_')
@@ -195,7 +197,7 @@ class HeavyDutyBacktester:
              print(f"   📂 [{sym}] Data Exists -> Skipping.")
              return
 
-        print(f"   ⚙️ [CPU] Analyzing {sym} (Full Stack Injection)...", flush=True)
+        print(f"   ⚙️ [CPU] Analyzing {sym} (Full Stack: Titan+Oracle+Sniper+Hydra)...", flush=True)
         t0 = time.time()
 
         df_1m = pd.DataFrame(candles, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
@@ -253,8 +255,6 @@ class HeavyDutyBacktester:
 
         # --- 4. Main Simulation Loop ---
         for i, current_time in enumerate(final_valid_indices):
-            
-            # Progress Stamp
             if i > 0 and i % 1000 == 0:
                 percent = (i / total_signals) * 100
                 print(f"     ⏳ [{sym}] Processing... {i}/{total_signals} ({percent:.1f}%)", flush=True)
@@ -263,11 +263,8 @@ class HeavyDutyBacktester:
             
             # Sync Indices
             idx_1m = np.searchsorted(fast_1m['timestamp'], ts_val)
-            
-            # ✅ FIX: Define idx_5m explicitly here for Legacy V2/V3 usage
-            idx_5m = np.searchsorted(numpy_htf['5m']['timestamp'], ts_val)
-            
             idx_1h = np.searchsorted(numpy_htf['1h']['timestamp'], ts_val)
+            idx_5m = np.searchsorted(numpy_htf['5m']['timestamp'], ts_val) # ✅ FIXED
             idx_15m = np.searchsorted(numpy_htf['15m']['timestamp'], ts_val)
             idx_4h = np.searchsorted(numpy_htf['4h']['timestamp'], ts_val)
             
@@ -299,7 +296,6 @@ class HeavyDutyBacktester:
                     elif col == 'sim_mc_score': val = 0.5
                     elif col == 'sim_pattern_score': val = 0.5
                     o_vec.append(val)
-                
                 try:
                     o_pred = oracle_dir_model.predict(np.array(o_vec).reshape(1, -1))[0]
                     oracle_conf = float(o_pred[0]) if isinstance(o_pred, (list, np.ndarray)) else float(o_pred)
@@ -318,13 +314,12 @@ class HeavyDutyBacktester:
                         s_vec.append(l_val)
                     else:
                         s_vec.append(0.0)
-                
                 try:
                     s_preds = [m.predict(np.array(s_vec).reshape(1, -1))[0] for m in sniper_models]
                     sniper_score = np.mean(s_preds)
                 except: pass
 
-            # === RISK SIMULATION (Hydra/Legacy - Minute by Minute) ===
+            # === RISK SIMULATION ===
             entry_price = fast_1m['close'][idx_1m]
             highest_price = entry_price 
             
@@ -338,7 +333,7 @@ class HeavyDutyBacktester:
                 curr_ts = int(fast_1m['timestamp'][c_idx])
                 if curr_price > highest_price: highest_price = curr_price
                 
-                # A. Hydra Injection
+                # A. Hydra
                 if hydra_models:
                     atr_val = fast_1m['atr'][c_idx]
                     sl_dist = 1.5 * atr_val if atr_val > 0 else entry_price * 0.015
@@ -366,7 +361,7 @@ class HeavyDutyBacktester:
                         if pg > max_hydra_giveback: max_hydra_giveback = pg
                     except: pass
 
-                # B. Legacy (Full Reconstruction)
+                # B. Legacy (Full Logic)
                 if legacy_v2 or legacy_v3:
                     c_5m_idx = idx_5m + (c_idx - idx_1m) // 5
                     if c_5m_idx >= len(numpy_htf['5m']['rsi']): c_5m_idx = len(numpy_htf['5m']['rsi']) - 1
@@ -440,24 +435,30 @@ class HeavyDutyBacktester:
         del frames, fast_1m, numpy_htf
         gc.collect()
 
+    # ==============================================================
+    # PHASE 1: Main Loop
+    # ==============================================================
     async def generate_truth_data(self):
         if self.force_start_date and self.force_end_date:
             dt_start = datetime.strptime(self.force_start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
             dt_end = datetime.strptime(self.force_end_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
             start_time_ms = int(dt_start.timestamp() * 1000)
             end_time_ms = int(dt_end.timestamp() * 1000)
-            print(f"\n🚜 [Phase 1] Processing Era: {self.force_start_date} -> {self.force_end_date}")
-        else: return 
-        
-        for sym in self.TARGET_COINS:
-            try:
-                candles = await self._fetch_all_data_fast(sym, start_time_ms, end_time_ms)
-                if candles:
-                    await self._process_data_in_memory(sym, candles, start_time_ms, end_time_ms)
-            except Exception as e:
-                print(f"   ❌ SKIP {sym}: {e}", flush=True)
-            gc.collect()
+            print(f"\n🚜 [Phase 1] Processing Forced Era: {self.force_start_date} -> {self.force_end_date}")
+            for sym in self.TARGET_COINS:
+                try:
+                    candles = await self._fetch_all_data_fast(sym, start_time_ms, end_time_ms)
+                    if candles: await self._process_data_in_memory(sym, candles, start_time_ms, end_time_ms)
+                except Exception as e: print(f"   ❌ SKIP {sym}: {e}", flush=True)
+                gc.collect()
+        else:
+            # If no forced date, we might rely on the Scenario Loop in run_strategic_optimization_task calling this.
+            # But the Scenario Loop sets force_start_date.
+            pass
 
+    # ==============================================================
+    # PHASE 2: Optimization (Grid Search)
+    # ==============================================================
     @staticmethod
     def _worker_optimize(combinations_batch, scores_files, initial_capital, fees_pct, max_slots):
         results = []
@@ -482,6 +483,7 @@ class HeavyDutyBacktester:
             max_drawdown = 0.0
 
             for ts, group in grouped_by_time:
+                # EXIT
                 active = list(wallet["positions"].keys())
                 current_prices = {row['symbol']: row['close'] for _, row in group.iterrows()}
                 
@@ -496,18 +498,18 @@ class HeavyDutyBacktester:
                         
                         pnl = (curr - pos['entry']) / pos['entry']
                         if is_crash or pnl > 0.04 or pnl < -0.02:
-                            gross = pos['size'] * pnl
-                            net = gross - (pos['size'] * fees_pct * 2)
-                            wallet['balance'] += (pos['size'] + net)
+                            wallet['balance'] += pos['size'] * (1 + pnl - (fees_pct*2))
                             wallet['allocated'] -= pos['size']
                             del wallet['positions'][sym]
-                            wallet['trades_history'].append({'pnl': net, 'pct': pnl})
+                            wallet['trades_history'].append({'pnl': pnl})
 
+                # Stats Update
                 total_eq = wallet['balance'] + wallet['allocated']
                 if total_eq > peak_balance: peak_balance = total_eq
                 dd = (peak_balance - total_eq) / peak_balance
                 if dd > max_drawdown: max_drawdown = dd
 
+                # ENTRY
                 if len(wallet['positions']) < max_slots:
                     for _, row in group.iterrows():
                         if row['symbol'] in wallet['positions']: continue
@@ -524,41 +526,31 @@ class HeavyDutyBacktester:
                             wallet['balance'] -= size
                             wallet['allocated'] += size
             
+            # Stats
             final_bal = wallet['balance'] + wallet['allocated']
             net_profit = final_bal - initial_capital
-            
             trades = wallet['trades_history']
             total_t = len(trades)
             win_count = len([t for t in trades if t['pnl'] > 0])
             loss_count = len([t for t in trades if t['pnl'] <= 0])
             win_rate = (win_count / total_t * 100) if total_t > 0 else 0
-            
             max_win = max([t['pnl'] for t in trades]) if trades else 0
             max_loss = min([t['pnl'] for t in trades]) if trades else 0
             
-            max_win_streak = 0; max_loss_streak = 0; curr_w = 0; curr_l = 0
-            for t in trades:
-                if t['pnl'] > 0:
-                    curr_w += 1; curr_l = 0
-                    if curr_w > max_win_streak: max_win_streak = curr_w
-                else:
-                    curr_l += 1; curr_w = 0
-                    if curr_l > max_loss_streak: max_loss_streak = curr_l
-
             results.append({
                 'config': config, 'final_balance': final_bal, 'net_profit': net_profit,
                 'total_trades': total_t, 'win_count': win_count, 'loss_count': loss_count,
                 'win_rate': win_rate, 'max_single_win': max_win, 'max_single_loss': max_loss,
-                'max_win_streak': max_win_streak, 'max_loss_streak': max_loss_streak,
                 'max_drawdown': max_drawdown * 100
             })
             
         return results
 
     async def run_optimization(self, target_regime="RANGE"):
+        # Note: generate_truth_data is called by the Strategy Loop wrapper now
+        # so we process data for the specific era set in set_date_range
         await self.generate_truth_data()
         
-        w_titan_range = [0.5]
         oracle_range = [0.5, 0.6, 0.7]
         sniper_range = [0.4, 0.5, 0.6]
         hydra_range = [0.75, 0.85, 0.95]
@@ -572,11 +564,9 @@ class HeavyDutyBacktester:
             })
             
         current_period_files = [os.path.join(CACHE_DIR, f) for f in os.listdir(CACHE_DIR) if f.endswith('_scores.pkl')]
-        if not current_period_files: 
-            print("❌ No data found.")
-            return None, None
+        if not current_period_files: return None, None
         
-        print(f"\n🧩 [Phase 2] Optimizing {len(combinations)} Configs (Full Stack)...")
+        print(f"\n🧩 [Phase 2] Optimizing {len(combinations)} Configs (Full Stack) for {target_regime}...")
         best_res = self._worker_optimize(combinations, current_period_files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS)
         
         if not best_res: return None, None
@@ -615,14 +605,29 @@ async def run_strategic_optimization_task():
     try:
         hub = AdaptiveHub(r2); await hub.initialize()
         optimizer = HeavyDutyBacktester(dm, proc)
-        # Set Date Range
-        optimizer.set_date_range("2024-03-01", "2024-03-05") 
         
-        best_cfg, best_stats = await optimizer.run_optimization(target_regime="RANGE")
-        if best_cfg:
-             hub.submit_challenger("RANGE", best_cfg, best_stats)
-             await hub._save_state_to_r2()
-             print("✅ [System] DNA Updated.")
+        # ✅ RESTORED: The Multi-Regime Strategic Loop
+        scenarios = [
+            {"regime": "BULL", "start": "2024-01-01", "end": "2024-03-30"},
+            {"regime": "BEAR", "start": "2023-08-01", "end": "2023-09-15"},
+            {"regime": "DEAD", "start": "2023-06-01", "end": "2023-08-01"},
+            {"regime": "RANGE", "start": "2024-07-01", "end": "2024-09-30"}
+        ]
+        
+        for scen in scenarios:
+            target = scen["regime"]
+            optimizer.set_date_range(scen["start"], scen["end"])
+            
+            # Run opt
+            best_cfg, best_stats = await optimizer.run_optimization(target_regime=target)
+            
+            # Save
+            if best_cfg:
+                hub.submit_challenger(target, best_cfg, best_stats)
+                
+        await hub._save_state_to_r2()
+        print("✅ [System] ALL Strategic DNA Updated & Saved.")
+        
     finally:
         await dm.close()