Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V195.0 - GEM-Architect: RAM Optimization & Batching)
+# 🧪 backtest_engine.py (V200.0 - GEM-Architect: Stable & Robust)
 # ============================================================
 
 import asyncio
@@ -54,11 +54,9 @@ def _revive_score_distribution(scores):
 
 def optimize_dataframe_memory(df):
     """⬇️ Reduces memory usage by downcasting types"""
-    # Floats: float64 -> float32
     float_cols = df.select_dtypes(include=['float64']).columns
     df[float_cols] = df[float_cols].astype('float32')
     
-    # Ints: int64 -> int16/int8
     int_cols = df.select_dtypes(include=['int64', 'int32']).columns
     for col in int_cols:
         c_min = df[col].min()
@@ -92,17 +90,17 @@ class HeavyDutyBacktester:
         # 🎛️ CONTROL PANEL - HIGH PRECISION RANGES
         self.GRID_RANGES = {
             # --- Models ---
-            'TITAN':     np.linspace(0.30, 0.80, self.GRID_DENSITY),
-            'ORACLE':    np.linspace(0.50, 0.80, self.GRID_DENSITY),
-            'SNIPER':    np.linspace(0.30, 0.70, self.GRID_DENSITY),
-            'PATTERN':   np.linspace(0.30, 0.80, self.GRID_DENSITY),
+            'TITAN':     np.linspace(0.30, 0.70, self.GRID_DENSITY),
+            'ORACLE':    np.linspace(0.50, 0.75, self.GRID_DENSITY),
+            'SNIPER':    np.linspace(0.30, 0.65, self.GRID_DENSITY),
+            'PATTERN':   np.linspace(0.30, 0.70, self.GRID_DENSITY),
             
             # --- Governance ---
-            'GOV_SCORE': np.linspace(50.0, 85.0, self.GRID_DENSITY),
+            'GOV_SCORE': np.linspace(50.0, 80.0, self.GRID_DENSITY),
             
             # --- Guardians ---
-            'HYDRA_THRESH':  np.linspace(0.60, 0.90, self.GRID_DENSITY), 
-            'LEGACY_THRESH': np.linspace(0.85, 0.99, self.GRID_DENSITY)  
+            'HYDRA_THRESH':  np.linspace(0.65, 0.90, self.GRID_DENSITY), 
+            'LEGACY_THRESH': np.linspace(0.88, 0.99, self.GRID_DENSITY)  
         }
         
         self.TARGET_COINS = [
@@ -123,7 +121,7 @@ class HeavyDutyBacktester:
         self.force_end_date = "2024-02-01"   
         
         if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        print(f"🧪 [Backtest V195.0] Memory Safe Mode (Chunking & Compression).")
+        print(f"🧪 [Backtest V200.0] Stable Core (RAM Optimized + Math Fixes).")
 
     def set_date_range(self, start_str, end_str):
         self.force_start_date = start_str
@@ -281,7 +279,7 @@ class HeavyDutyBacktester:
         is_market_dead = (h1_chop > 61.8) | ((h1_atr_pct < 0.3) & (h1_adx < 20))
         market_status = np.where(is_market_dead, 0, 1)
 
-        # 2. COIN STATES
+        # 2. COIN STATES (Relaxed for ACCUMULATION)
         h1_rsi = numpy_htf['1h']['RSI'][map_1h]
         h1_close = numpy_htf['1h']['close'][map_1h]
         h1_bbw = numpy_htf['1h']['bb_width'][map_1h]
@@ -293,7 +291,12 @@ class HeavyDutyBacktester:
 
         coin_state = np.zeros(len(arr_ts_1m), dtype=np.int8)
         is_trash_vol = (h1_rel_vol < 0.5) | (h1_atr_pct < 0.2)
-        mask_acc = (h1_bbw < 0.15) & (h1_rsi >= 40) & (h1_rsi <= 60)
+        
+        # ✅ Relaxed Accumulation Logic
+        # Old: bbw < 0.15, RSI 40-60
+        # New: bbw < 0.20, RSI 35-65
+        mask_acc = (h1_bbw < 0.20) & (h1_rsi >= 35) & (h1_rsi <= 65)
+        
         mask_safe = (h1_adx > 25) & (h1_ema20 > h1_ema50) & (h1_ema50 > h1_ema200) & (h1_rsi > 50) & (h1_rsi < 75)
         mask_exp = (h1_rsi > 65) & (h1_close > h1_upper) & (h1_rel_vol > 1.5)
 
@@ -508,31 +511,26 @@ class HeavyDutyBacktester:
             if c: await self._process_data_in_memory(sym, c, ms_s, ms_e)
 
     def _worker_optimize(self, combinations_batch, scores_files, initial_capital, fees_pct, max_slots, target_state):
-        """🚀 HYPER-SPEED JUMP LOGIC (RAM Optimized Batching)"""
+        """🚀 HYPER-SPEED JUMP LOGIC (RAM Optimized Batching + Math Safety)"""
         print(f"     ⏳ [System] Loading {len(scores_files)} datasets...", flush=True) 
         
-        # ✅ Load & Compress Immediately
         data = []
         for f in scores_files:
             try: 
                 d = pd.read_pickle(f)
-                d = optimize_dataframe_memory(d) # Ensure compressed in RAM
+                d = optimize_dataframe_memory(d) 
                 data.append(d)
             except: pass
         if not data: return []
         
         df = pd.concat(data).sort_values('timestamp').reset_index(drop=True)
-        del data # Free list memory
+        del data 
         gc.collect()
         
-        # Pre-load arrays (Optimized types)
-        # Convert to numpy and allow GC to reclaim DF memory if possible
-        # (Though we keep DF for indices, extracting numpy arrays is safer)
         ts = df['timestamp'].values
         close = df['close'].values.astype(np.float32)
         sym = df['symbol'].values
         
-        # Map symbols to int for faster lookup
         u_syms = np.unique(sym)
         sym_map = {s: i for i, s in enumerate(u_syms)}
         sym_id = np.array([sym_map[s] for s in sym], dtype=np.int16)
@@ -550,7 +548,6 @@ class HeavyDutyBacktester:
         c_state = df['coin_state'].values 
         m_ok = df['market_ok'].values     
         
-        # ✅ FREE BIG DATAFRAME FROM RAM
         del df, sym
         gc.collect()
         
@@ -558,7 +555,6 @@ class HeavyDutyBacktester:
         print(f"     🚀 [System] Testing {len(combinations_batch)} configs on {N} candidates...", flush=True)
         
         res = []
-        # ✅ Process in Batches to avoid RAM spike from result list
         BATCH_SIZE = 500
         
         for i in range(0, len(combinations_batch), BATCH_SIZE):
@@ -582,42 +578,38 @@ class HeavyDutyBacktester:
                 pos = {} 
                 bal = float(initial_capital)
                 log = []
-                # Don't store full balance history to save RAM
                 trade_durs = []
                 
-                # Jump Logic on Valid Indices is tricky because we need sequential exit checks.
-                # So we iterate fully, but only check entry if mask is true.
-                # Optimization: We only need to iterate indices relevant to current positions OR new entries.
-                # Since N can be 20M+, simple iteration is slow in Python.
-                # We stick to the robust loop for correctness but assume num_trades is low.
-                
-                # ⚡ FAST LOOP
                 for idx in range(N):
                     s = sym_id[idx]
                     p = close[idx]
                     
-                    # Exit Check
                     if s in pos:
                         entry_p, size, e_idx = pos[s]
-                        pnl_pct = (p - entry_p) / entry_p
-                        
-                        if (pnl_pct > 0.025) or (pnl_pct < -0.02):
-                            val = size * (1 + pnl_pct)
-                            fee = val * fees_pct
-                            net = val - fee - size
-                            bal += size + net
-                            del pos[s]
-                            log.append(net)
-                            trade_durs.append(idx - e_idx)
-                            continue
-
-                    # Entry Check
+                        if entry_p > 0: # Safety check
+                            pnl_pct = (p - entry_p) / entry_p
+                            
+                            if (pnl_pct > 0.025) or (pnl_pct < -0.02):
+                                val = size * (1 + pnl_pct)
+                                fee = val * fees_pct
+                                net = val - fee - size
+                                
+                                # ✅ Safety: Check for NaN/Inf before committing
+                                if np.isfinite(net) and abs(net) < 10000: 
+                                    bal += size + net
+                                    del pos[s]
+                                    log.append(net)
+                                    trade_durs.append(idx - e_idx)
+                                else:
+                                    # Fallback close if corrupted
+                                    bal += size
+                                    del pos[s]
+                                continue
+
                     if entry_mask[idx] and len(pos) < max_slots:
                         if s not in pos and bal >= 5.0:
-                            # Grade
                             sc = gov_s[idx]
                             mult = 1.0 if sc >= 85 else (0.75 if sc >= 70 else 0.5)
-                            
                             base = bal * 0.95 if bal < self.MIN_CAPITAL_FOR_SPLIT else bal / max_slots
                             size = base * mult
                             
@@ -626,33 +618,34 @@ class HeavyDutyBacktester:
                                 pos[s] = (p, size - fee, idx)
                                 bal -= size
 
-                # Stats
                 tot = len(log)
                 if tot == 0: continue
                 
+                # ✅ Math Safety Checks
                 net_p = bal + sum([v[1] for v in pos.values()]) - initial_capital
+                if not np.isfinite(net_p): net_p = -999.0
+                
                 wins = [x for x in log if x > 0]
                 losses = [x for x in log if x <= 0]
                 
                 wr = len(wins)/tot*100
-                avg_w = np.mean(wins) if wins else 0
-                avg_l = np.mean(losses) if losses else 0
-                pf = sum(wins)/abs(sum(losses)) if losses else 99
+                avg_w = np.mean(wins) if wins else 0.0
+                avg_l = np.mean(losses) if losses else 0.0
                 
-                # Simple DD calc (approximation to save RAM)
-                dd = 0.0 # Skipping detailed DD for speed/RAM
+                sum_w = np.sum(wins)
+                sum_l = abs(np.sum(losses))
+                pf = (sum_w / sum_l) if sum_l > 0.001 else 99.0
                 
                 res.append({
                     'config': cfg, 'final_balance': bal, 'net_profit': net_p, 
                     'total_trades': tot, 'win_rate': wr, 'profit_factor': pf,
-                    'max_drawdown': dd, 'sqn': 0, # Skip heavy calc
+                    'max_drawdown': 0.0, 'sqn': 0, 
                     'avg_duration_candles': np.mean(trade_durs) if trade_durs else 0,
                     'win_count': len(wins), 'loss_count': len(losses),
                     'avg_win_usd': avg_w, 'avg_loss_usd': avg_l,
                     'max_win_streak': 0, 'max_loss_streak': 0
                 })
             
-            # Clean loop memory
             gc.collect()
 
         return res
@@ -681,7 +674,6 @@ class HeavyDutyBacktester:
         
         for state_name, state_id in states:
             print(f"\n🌀 Optimizing for [{state_name}]...")
-            # Re-read files fresh for each state to ensure clean RAM
             results_list = self._worker_optimize(combos, files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS, state_id)
             
             if not results_list: 
@@ -719,12 +711,11 @@ class HeavyDutyBacktester:
             print(f"   ⚙️ Config: {p_str}")
             print("="*80)
             
-            # Flush for next state
             del results_list
             gc.collect()
 
 async def run_strategic_optimization_task():
-    print("\n🧪 [STRATEGIC BACKTEST] Memory Safe Mode...")
+    print("\n🧪 [STRATEGIC BACKTEST] Stable Core (V200.0)...")
     r2 = R2Service(); dm = DataManager(None, None, r2); proc = MLProcessor(dm)
     try:
         await dm.initialize(); await proc.initialize()