Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V122.0 - GEM-Architect: Full Spectrum)
+# 🧪 backtest_engine.py (V122.1 - GEM-Architect: Method Scope Fix)
 # ============================================================
 
 import asyncio
@@ -34,10 +34,10 @@ logging.getLogger('ml_engine').setLevel(logging.WARNING)
 CACHE_DIR = "backtest_real_scores"
 
 # ============================================================
-# 🔧 1. PATTERN ENGINE LOGIC (EXACT REPLICA)
+# 🔧 1. FEATURE ENGINEERING REPLICAS (GLOBAL SCOPE)
 # ============================================================
+
 def _zv(x):
-    """Z-Score Normalization (Safe)"""
     with np.errstate(divide='ignore', invalid='ignore'):
         x = np.asarray(x, dtype="float32")
         m = np.nanmean(x, axis=0)
@@ -45,39 +45,27 @@ def _zv(x):
         return np.nan_to_num((x - m) / s, nan=0.0)
 
 def _transform_window_for_pattern(df_window):
-    """
-    تحويل نافذة (200 شمعة) إلى متجه ميزات كما في patterns.py تماماً.
-    """
     try:
-        # Extract numpy arrays
         c = df_window['close'].values.astype('float32')
         o = df_window['open'].values.astype('float32')
         h = df_window['high'].values.astype('float32')
         l = df_window['low'].values.astype('float32')
         v = df_window['volume'].values.astype('float32')
 
-        # 1. Base Features
-        base = np.stack([o, h, l, c, v], axis=1) # (200, 5)
+        base = np.stack([o, h, l, c, v], axis=1)
         base_z = _zv(base)
 
-        # 2. Extra
         lr = np.zeros_like(c); lr[1:] = np.diff(np.log1p(c))
         rng = (h - l) / (c + 1e-9)
         extra = np.stack([lr, rng], axis=1)
         extra_z = _zv(extra)
 
-        # 3. Indicators (Simplified Calculation for Speed using TA-Lib vectorization if possible, else manual)
-        # Using pandas_ta on small window is slow. We use numpy optimized versions.
-        # EMA
         def _ema(arr, n):
-            alpha = 2/(n+1)
-            # Fast recursive implementation or pandas ewm
             return pd.Series(arr).ewm(span=n, adjust=False).mean().values
 
         ema9 = _ema(c, 9); ema21 = _ema(c, 21); ema50 = _ema(c, 50); ema200 = _ema(c, 200)
         slope21 = np.gradient(ema21); slope50 = np.gradient(ema50)
         
-        # RSI (Numpy version approx)
         delta = np.diff(c, prepend=c[0])
         up, down = delta.copy(), delta.copy()
         up[up < 0] = 0; down[down > 0] = 0
@@ -86,30 +74,18 @@ def _transform_window_for_pattern(df_window):
         rs = roll_up / (roll_down + 1e-9)
         rsi = 100.0 - (100.0 / (1.0 + rs))
 
-        # ATR & BB & OBV (Simplified for speed match)
-        # ... (Assuming simplified indicators for pattern matching structure)
-        
-        # Stack indicators
         indicators = np.stack([ema9, ema21, ema50, ema200, slope21, slope50, rsi], axis=1)
-        # Padding to match 12 features if needed (Patterns.py has 12)
-        # We fill zeros for heavy ones like MACD/OBV to save time, relying on EMA structure
         padding = np.zeros((200, 5), dtype='float32') 
         indicators_full = np.concatenate([indicators, padding], axis=1)
         indicators_z = _zv(indicators_full)
 
-        # 4. Flatten
-        X_seq = np.concatenate([base_z, extra_z, indicators_z], axis=1) # (200, Features)
+        X_seq = np.concatenate([base_z, extra_z, indicators_z], axis=1)
         X_flat = X_seq.flatten()
-
-        # 5. Stat (MC/Hurst - Mocked for speed as 0.5)
         X_stat = np.array([0.5, 0.0, 0.5], dtype="float32")
 
         return np.concatenate([X_flat, X_stat])
     except: return None
 
-# ============================================================
-# 🔧 2. SNIPER FEATURES (EXACT REPLICA)
-# ============================================================
 def _z_score_rolling(x, w=500): 
     r = x.rolling(w).mean(); s = x.rolling(w).std().replace(0, np.nan); return ((x - r) / s).fillna(0)
 
@@ -150,9 +126,6 @@ def calculate_sniper_features_exact(df):
     
     return df.fillna(0)
 
-# ============================================================
-# 🔧 3. TITAN FEATURES (REAL)
-# ============================================================
 def calculate_titan_features_real(df):
     df = df.copy()
     df['RSI'] = ta.rsi(df['close'], 14)
@@ -191,29 +164,47 @@ class HeavyDutyBacktester:
             for f in glob.glob(os.path.join(CACHE_DIR, "*")): os.remove(f)
         else: os.makedirs(CACHE_DIR)
             
-        print(f"🧪 [Backtest V122.0] Full Spectrum Mode. Checking Engines...")
+        print(f"🧪 [Backtest V122.1] Method Scope Fix. Checking Engines...")
         self._check_engines()
 
     def _check_engines(self):
         status = []
-        if self.proc.titan and self.proc.titan.model: status.append("Titan")
-        if self.proc.pattern_engine and self.proc.pattern_engine.models: status.append("Patterns")
-        else: print("⚠️ Pattern Models Missing (Will use neutral)")
-        if self.proc.oracle: status.append("Oracle")
-        if self.proc.sniper: status.append("Sniper")
-        if self.proc.guardian_hydra: status.append("Hydra")
+        if self.proc.titan and self.proc.titan.model: status.append("Titan(Real)")
+        else: print("❌ Titan Model MISSING")
+        if self.proc.pattern_engine and self.proc.pattern_engine.models: status.append("Patterns(Real)")
+        else: print("⚠️ Pattern Models MISSING")
+        if self.proc.oracle: status.append("Oracle(Real)")
+        if self.proc.sniper: status.append("Sniper(Real)")
+        if self.proc.guardian_hydra: status.append("Hydra(Real)")
         print(f"   ✅ Engines Ready: {', '.join(status)}")
 
     def set_date_range(self, start_str, end_str):
         self.force_start_date = start_str; self.force_end_date = end_str
 
+    # ✅ FIXED: Helper methods are now INSIDE the class
     def _smart_predict(self, model, X):
+        """Forces predict_proba and handles shape mismatch"""
         try:
+            # 1. Try Proba (For Classifiers)
             if hasattr(model, "predict_proba"):
                 raw = model.predict_proba(X)
-                return raw[:, -1] if raw.ndim == 2 else raw
+                if raw.ndim == 2:
+                    return raw[:, -1] # Positive/Buy Class
+                return raw 
+            
+            # 2. Fallback to predict
             return model.predict(X)
-        except: return np.zeros(len(X))
+        except:
+            return np.zeros(len(X) if hasattr(X, '__len__') else 0)
+
+    def _extract_probs(self, raw_preds):
+        """Standardizes output to 1D probability array"""
+        if isinstance(raw_preds, list): raw_preds = np.array(raw_preds)
+        if raw_preds.ndim == 1: return raw_preds
+        elif raw_preds.ndim == 2:
+            if raw_preds.shape[1] >= 2: return raw_preds[:, -1]
+            return raw_preds.flatten()
+        return raw_preds.flatten()
 
     async def _fetch_all_data_fast(self, sym, start_ms, end_ms):
         print(f"   ⚡ [Network] Downloading {sym}...", flush=True)
@@ -257,9 +248,7 @@ class HeavyDutyBacktester:
         df_1m.set_index('datetime', inplace=True)
         df_1m = df_1m.sort_index()
 
-        # -----------------------------------------------------------
         # 1️⃣ SNIPER & L1 SCORING (L4/L1)
-        # -----------------------------------------------------------
         df_sniper = calculate_sniper_features_exact(df_1m)
         df_sniper['rel_vol'] = df_sniper['volume'] / (df_sniper['volume'].rolling(50).mean() + 1e-9)
         df_sniper['l1_score'] = (df_sniper['rel_vol'] * 10) + ((df_sniper['atr']/df_sniper['close']) * 1000)
@@ -270,71 +259,45 @@ class HeavyDutyBacktester:
 
         print(f"     🎯 Candidates: {len(df_candidates)}. Running Deep Inference...", flush=True)
 
-        # -----------------------------------------------------------
-        # 2️⃣ PATTERNS (L2) - RUN ON 15M -> MAP TO 1M
-        # -----------------------------------------------------------
+        # 2️⃣ PATTERNS (L2)
         res_patterns = np.full(len(df_candidates), 0.5)
         pattern_models = getattr(self.proc.pattern_engine, 'models', {})
-        
         if pattern_models and '15m' in pattern_models:
             try:
-                # Resample to 15m
                 df_15m = df_1m.resample('15T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
-                
-                # Create Windows (200 candles)
-                # Need to iterate efficiently
                 pat_scores_15m = np.full(len(df_15m), 0.5)
-                # Only compute for 15m candles that cover candidate times
-                
-                # Simplified: Loop through 15m (8k candles) is fast enough
-                pat_inputs = []
-                valid_15m_idxs = []
+                pat_inputs = []; valid_15m_idxs = []
                 
                 for i in range(200, len(df_15m)):
                     window = df_15m.iloc[i-200:i]
                     vec = _transform_window_for_pattern(window)
                     if vec is not None:
-                        pat_inputs.append(vec)
-                        valid_15m_idxs.append(i)
+                        pat_inputs.append(vec); valid_15m_idxs.append(i)
                 
                 if pat_inputs:
                     X_pat = np.array(pat_inputs)
                     pat_preds = self._smart_predict(pattern_models['15m'], xgb.DMatrix(X_pat))
                     pat_scores_15m[valid_15m_idxs] = pat_preds
                 
-                # Map 15m scores back to candidates (Forward Fill)
-                cand_ts = df_candidates['timestamp'].values
-                # 15m timestamps (end of candle approx)
                 ts_15m = df_15m.index.astype(np.int64) // 10**6
-                
-                # Searchsorted to find last closed 15m candle
-                map_idxs = np.searchsorted(ts_15m, cand_ts) - 1
-                map_idxs = np.clip(map_idxs, 0, len(pat_scores_15m)-1)
-                res_patterns = pat_scores_15m[map_idxs]
-                
+                map_idxs = np.searchsorted(ts_15m, df_candidates['timestamp'].values) - 1
+                res_patterns = pat_scores_15m[np.clip(map_idxs, 0, len(pat_scores_15m)-1)]
             except Exception as e: print(f"Patterns Error: {e}")
 
-        # -----------------------------------------------------------
-        # 3️⃣ TITAN (L2) - RESAMPLE 5M
-        # -----------------------------------------------------------
+        # 3️⃣ TITAN (L2)
         res_titan = np.full(len(df_candidates), 0.5)
         if self.proc.titan and self.proc.titan.model:
             try:
                 df_5m = df_1m.resample('5T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
                 df_5m_feat = calculate_titan_features_real(df_5m).add_prefix('5m_')
-                
-                cand_ts = df_candidates['timestamp'].values
                 ts_5m = df_5m.index.astype(np.int64) // 10**6
-                map_idxs = np.clip(np.searchsorted(ts_5m, cand_ts) - 1, 0, len(df_5m_feat)-1)
-                
+                map_idxs = np.clip(np.searchsorted(ts_5m, df_candidates['timestamp'].values) - 1, 0, len(df_5m_feat)-1)
                 feats = self.proc.titan.feature_names
                 X_titan = df_5m_feat.iloc[map_idxs].reindex(columns=feats, fill_value=0).values
                 res_titan = self.proc.titan.model.predict(xgb.DMatrix(X_titan, feature_names=feats))
             except Exception as e: print(f"Titan Error: {e}")
 
-        # -----------------------------------------------------------
         # 4️⃣ SNIPER (L4)
-        # -----------------------------------------------------------
         res_sniper = np.full(len(df_candidates), 0.5)
         sniper_models = getattr(self.proc.sniper, 'models', [])
         if sniper_models:
@@ -343,28 +306,17 @@ class HeavyDutyBacktester:
                 for f in feats: 
                     if f not in df_candidates.columns: df_candidates[f] = 0.0
                 X_snip = df_candidates[feats].values
+                # ✅ Calling internal method self._smart_predict
                 preds = [self._extract_probs(self._smart_predict(m, X_snip)) for m in sniper_models]
                 res_sniper = np.mean(preds, axis=0)
             except Exception as e: print(f"Sniper Error: {e}")
 
-        # -----------------------------------------------------------
         # 5️⃣ ORACLE (L3)
-        # -----------------------------------------------------------
         res_oracle = np.full(len(df_candidates), 0.5)
         oracle_model = getattr(self.proc.oracle, 'model_direction', None)
         if oracle_model:
             try:
-                # We reuse res_titan and res_patterns here!
                 oracle_feats = getattr(self.proc.oracle, 'feature_cols', [])
-                # Placeholder HTF logic (reusing L2/L4 data for speed as 'best effort')
-                # In real system, Processor gathers these. Here we simulate the gather.
-                # Titan Score -> 'sim_titan_score'
-                # Pattern Score -> 'sim_pattern_score'
-                
-                # Build Oracle Vector
-                # We cheat slightly by filling HTF cols with zeros (neutral) or proxies 
-                # because fetching full 1h/4h features for 128k candles is too heavy.
-                # But we provide the Critical Scores.
                 X_orc_df = pd.DataFrame(0.0, index=range(len(df_candidates)), columns=oracle_feats)
                 if 'sim_titan_score' in X_orc_df: X_orc_df['sim_titan_score'] = res_titan
                 if 'sim_pattern_score' in X_orc_df: X_orc_df['sim_pattern_score'] = res_patterns
@@ -373,15 +325,11 @@ class HeavyDutyBacktester:
                 res_oracle = self._extract_probs(self._smart_predict(oracle_model, X_orc_df.values))
             except Exception as e: print(f"Oracle Error: {e}")
 
-        # -----------------------------------------------------------
         # 6️⃣ HYDRA & LEGACY (L0)
-        # -----------------------------------------------------------
         res_hydra_risk = np.zeros(len(df_candidates)); res_hydra_time = np.zeros(len(df_candidates), dtype=int)
-        res_legacy_risk = np.zeros(len(df_candidates))
         
         hydra_models = getattr(self.proc.guardian_hydra, 'models', {})
         if hydra_models and 'crash' in hydra_models:
-            # Sliding window approach
             try:
                 global_hydra_feats = np.column_stack([
                     df_sniper['rsi_14'], df_sniper['rsi_14'], df_sniper['rsi_14'],
@@ -390,14 +338,11 @@ class HeavyDutyBacktester:
                 ]).astype(np.float32)
                 
                 window_view = sliding_window_view(global_hydra_feats, 240, axis=0).transpose(0, 2, 1)
-                
-                # Map indices
                 c_idxs = np.searchsorted(df_sniper.index, df_candidates.index)
                 valid_s = c_idxs + 1
                 valid_mask_h = valid_s < (len(global_hydra_feats) - 240)
                 final_s = valid_s[valid_mask_h]; res_idxs = np.where(valid_mask_h)[0]
                 
-                # Batch Loop
                 for i in range(0, len(final_s), 5000):
                     b_idxs = final_s[i:i+5000]; r_idxs = res_idxs[i:i+5000]
                     static = window_view[b_idxs]
@@ -411,7 +356,6 @@ class HeavyDutyBacktester:
                     max_pnl = (np.maximum.accumulate(s_c, axis=1) - entry)/dist
                     atr_p = s_atr/(s_c+1e-9)
                     
-                    # Stack
                     zeros = np.zeros((B, 240)); ones = np.ones((B, 240)); t = np.tile(np.arange(1, 241), (B, 1))
                     X = np.stack([
                         static[:,0], static[:,1], static[:,2], static[:,3], static[:,4],
@@ -422,9 +366,7 @@ class HeavyDutyBacktester:
                     res_hydra_risk[r_idxs] = np.max(preds, axis=1)
             except: pass
 
-        # -----------------------------------------------------------
         # 7️⃣ FINAL ASSEMBLY
-        # -----------------------------------------------------------
         print(f"     📊 [Stats] Titan:{res_titan.mean():.2f} | Patterns:{res_patterns.mean():.2f} | Sniper:{res_sniper.mean():.2f} | Oracle:{res_oracle.mean():.2f}")
         
         ai_df = pd.DataFrame({
@@ -437,7 +379,7 @@ class HeavyDutyBacktester:
             'l1_score': df_candidates['l1_score'],
             'risk_hydra_crash': res_hydra_risk,
             'time_hydra_crash': res_hydra_time,
-            'risk_legacy_v2': res_legacy_risk,
+            'risk_legacy_v2': 0.0,
             'time_legacy_panic': 0
         })
 
@@ -493,9 +435,6 @@ class HeavyDutyBacktester:
                 # Exit
                 if s in pos:
                     entry = pos[s][0]; h_r = pos[s][1]
-                    # Check hydra risk vs config thresh (dynamic exit)
-                    # For backtest, simplified: if risk > thresh AND current PnL < -1% (validation)
-                    # Or just risk > thresh
                     crash = (h_r > cfg['hydra_thresh']) 
                     pnl = (p - entry)/entry
                     
@@ -526,21 +465,24 @@ class HeavyDutyBacktester:
         await self.generate_truth_data()
         
         # Grid
-        # Using wider ranges to ensure we find *something*
         oracle_r = np.linspace(0.3, 0.7, 3); sniper_r = np.linspace(0.2, 0.6, 3)
         hydra_r = [0.8, 0.9]; l1_r = [5.0, 10.0]
         
         combos = []
         for o, s, h, l1 in itertools.product(oracle_r, sniper_r, hydra_r, l1_r):
             combos.append({
-                'w_titan': 0.4, 'w_struct': 0.3, # Fixed weights for now
-                'l1_thresh': l1, 'oracle_thresh': o, 'sniper_thresh': s, 'hydra_thresh': h
+                'w_titan': 0.4, 'w_struct': 0.3,
+                'l1_thresh': l1, 'oracle_thresh': o, 'sniper_thresh': s, 'hydra_thresh': h, 'legacy_thresh': 0.95
             })
             
         files = glob.glob(os.path.join(CACHE_DIR, "*.pkl"))
         best_res = self._worker_optimize(combos, files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS)
-        if not best_res: return None, None
         
+        # ✅ FIX: Handle empty result
+        if not best_res:
+            print("⚠️ [Warning] No trades generated in any config.")
+            return None, {'net_profit': 0.0, 'win_rate': 0.0}
+
         print("\n" + "="*60)
         print(f"🏆 CHAMPION REPORT [{target_regime}]:")
         print(f"   💰 Final Balance:   ${best['final_balance']:,.2f}")
@@ -559,7 +501,6 @@ class HeavyDutyBacktester:
         print(f"   ⚖️ Weights: Titan={best['config']['w_titan']:.2f} | Patterns={best['config']['w_struct']:.2f} | L1={best['config']['l1_thresh']}")
         print("="*60)
         return best['config'], best
-
 async def run_strategic_optimization_task():
     print("\n🧪 [STRATEGIC BACKTEST] Full Spectrum Mode...")
     r2 = R2Service(); dm = DataManager(None, None, r2); proc = MLProcessor(dm)
@@ -570,9 +511,11 @@ async def run_strategic_optimization_task():
     opt = HeavyDutyBacktester(dm, proc)
     
     scenarios = [
-        {"regime": "BULL", "start": "2024-01-01", "end": "2024-03-30"},
-        # Add others...
-    ]
+            {"regime": "BULL", "start": "2024-01-01", "end": "2024-03-30"},
+            {"regime": "BEAR", "start": "2023-08-01", "end": "2023-09-15"},
+            {"regime": "DEAD", "start": "2023-06-01", "end": "2023-08-01"},
+            {"regime": "RANGE", "start": "2024-07-01", "end": "2024-09-30"}
+        ]
     
     for s in scenarios:
         opt.set_date_range(s["start"], s["end"])