Update backtest_engine.py

backtest_engine.py

@@ -1,10 +1,10 @@
 # ============================================================
-# 🧪 backtest_engine.py (V40.0 - GEM-Architect: Full Grid & Guardians)
+# 🧪 backtest_engine.py (V42.1 - GEM-Architect: Exact Threshold Mapping)
 # ============================================================
-# التحديثات الكبرى:
-# 1. Grid Search: توليد آلي لكل توليفات الأوزان والعتبات.
-# 2. Guardian Telemetry: تقرير مفصل عن تدخلات Hydra و Legacy.
-# 3. Stress Matrix: تحليل أعمق للمخاطر.
+# التحديثات:
+# 1. مطابقة تامة لمتغيرات processor.py (Hydra Crash/Giveback, Legacy V2/V3).
+# 2. توسيع الشبكة لتشمل هذه المتغيرات الدقيقة.
+# 3. تقرير يفصل أي "رأس" من رؤوس الحراس كان الأنشط.
 # ============================================================
 
 import asyncio
@@ -12,7 +12,7 @@ import pandas as pd
 import numpy as np
 import time
 import logging
-import itertools # 👈 مكتبة التباديل والتوافيق
+import itertools 
 from datetime import datetime
 from typing import Dict, Any, List
 
@@ -20,15 +20,23 @@ from ml_engine.processor import MLProcessor, SystemLimits
 from ml_engine.data_manager import DataManager
 from learning_hub.adaptive_hub import StrategyDNA
 
-# كتم الإشعارات الفرعية
+# كتم الضوضاء
 logging.getLogger('ml_engine.patterns').setLevel(logging.WARNING)
 
 class VirtualPortfolio:
     def __init__(self, initial_capital=1000.0):
         self.capital = initial_capital
         self.active_trades = {} 
-        self.stats = {"max_win_usd": 0.0, "max_loss_usd": 0.0, "max_drawdown_pct": 0.0}
-        self.guardian_log = {'hydra_crash': 0, 'hydra_giveback': 0, 'legacy_panic': 0, 'tp_hit': 0, 'sl_hit': 0}
+        self.stats = {
+            "max_win_usd": 0.0, "max_loss_usd": 0.0, 
+            "max_drawdown_pct": 0.0, "max_runup_pct": 0.0
+        }
+        # تفصيل أدق للحراس
+        self.guardian_log = {
+            'hydra_crash': 0, 'hydra_giveback': 0, 'hydra_stag': 0,
+            'legacy_v2': 0, 'legacy_v3': 0,
+            'tp': 0, 'sl': 0
+        }
         self.MAX_SLOTS_MAP = {'BULL': 6, 'BEAR': 3, 'RANGE': 5, 'DEAD': 2}
     
     def can_open_trade(self, regime):
@@ -43,10 +51,9 @@ class BacktestSimulator:
         self.dm = data_manager
         self.proc = processor
         self.history_cache = {} 
-        self.DAYS_TO_FETCH = 14 
+        self.DAYS_TO_FETCH = 7 
         self.CHUNK_LIMIT = 1000
         
-        # السلة الكاملة
         self.TARGET_COINS = [
             'BTC/USDT', 'ETH/USDT', 'BNB/USDT', 'SOL/USDT', 'XRP/USDT',
             'AVAX/USDT', 'ADA/USDT', 'LINK/USDT', 'NEAR/USDT', 'RUNE/USDT', 'INJ/USDT',
@@ -54,7 +61,7 @@ class BacktestSimulator:
             'XLM/USDT', 'TRX/USDT', 'LTC/USDT'
         ]
         
-        print("🧪 [Backtest Engine V40.0] Full Grid & Guardian System Initialized.")
+        print("🧪 [Backtest Engine V42.1] Exact Threshold Grid Initialized.")
 
     # ==========================================================================
     # 1. Data Loading
@@ -66,8 +73,10 @@ class BacktestSimulator:
         
         loaded_count = 0
         for sym in self.TARGET_COINS:
+            print(f"   ⬇️  {sym:<10}", end="", flush=True) 
             all_candles = []
             current_since = start_time_ms
+            
             while current_since < end_time_ms:
                 try:
                     candles = await self.dm.exchange.fetch_ohlcv(sym, '1m', since=current_since, limit=self.CHUNK_LIMIT)
@@ -89,6 +98,9 @@ class BacktestSimulator:
                 mask = (df.index >= pd.to_datetime(start_time_ms, unit='ms'))
                 self.history_cache[sym] = df.loc[mask]
                 loaded_count += 1
+                print(f" ✅ ({len(self.history_cache[sym])})")
+            else:
+                print(" ⚠️ No Data")
         
         print(f"✅ Data Load Complete. Cached {loaded_count}/{len(self.TARGET_COINS)} coins.")
 
@@ -176,27 +188,23 @@ class BacktestSimulator:
         return None
 
     # ==========================================================================
-    # 4. Simulation Loop (With Guardian Telemetry)
+    # 4. Simulation Loop (Exact Threshold Injection)
     # ==========================================================================
-    async def run_simulation(self, regime_name, weights, l1_thresh):
+    async def run_simulation(self, regime_name, weights, l1_thresh, hydra_crash, hydra_giveback, legacy_v2, legacy_v3):
         SystemLimits.CURRENT_REGIME = regime_name
         self.portfolio = VirtualPortfolio() 
-        trades_log = []
+        trades_log = [] 
 
-        if not self.history_cache: return {'final_capital': 0, 'win_rate': 0, 'trades': 0}
         ref_symbol = list(self.history_cache.keys())[0]
         full_index = self.history_cache[ref_symbol].index
-        
         start_idx = 6000
         end_idx = len(full_index) - 1
         current_idx = start_idx
         
-        # print(f"      ▶️ Sim...", end="", flush=True)
-
         while current_idx < end_idx:
             current_time = full_index[current_idx]
             
-            # --- أ. إدارة الصفقات (وتفعيل الحراس) ---
+            # --- أ. الحراس ---
             active_symbols = list(self.portfolio.active_trades.keys())
             for sym in active_symbols:
                 trade = self.portfolio.active_trades[sym]
@@ -205,11 +213,13 @@ class BacktestSimulator:
                     current_price = self.history_cache[sym].iloc[sym_idx]['close']
                 except: continue
                 
-                # تحديث القمم والقيعان للصفقة
                 if current_price > trade['highest_price']: trade['highest_price'] = current_price
                 if current_price < trade.get('lowest_price', trade['entry_price']): trade['lowest_price'] = current_price
 
-                # جلب Snapshot للحراس
+                # Max DD
+                dd = (trade['lowest_price'] - trade['entry_price']) / trade['entry_price']
+                if dd < self.portfolio.stats['max_drawdown_pct']: self.portfolio.stats['max_drawdown_pct'] = dd
+
                 snapshot = self.get_market_snapshot(sym, sym_idx)
                 if not snapshot: continue
 
@@ -220,39 +230,52 @@ class BacktestSimulator:
                     'time_in_trade_mins': (current_idx - trade['entry_idx'])
                 }
                 
-                # 🛡️ 1. Hydra Guardian (The 3 Heads)
+                # 🛡️ 1. Hydra (Exact Thresholds)
                 if self.proc.guardian_hydra:
+                    # حقن المتغيرات الدقيقة
+                    SystemLimits.HYDRA_CRASH_THRESH = hydra_crash
+                    SystemLimits.HYDRA_GIVEBACK_THRESH = hydra_giveback
+                    SystemLimits.HYDRA_STAGNATION_THRESH = 0.50 # ثابت حالياً
+                    
                     hydra_res = self.proc.guardian_hydra.analyze_position(sym, snapshot['1m'], snapshot['5m'], snapshot['15m'], trade_ctx)
                     if hydra_res['action'] in ['EXIT_HARD', 'EXIT_SOFT']:
-                        exit_reason = f"Hydra_{hydra_res['reason']}" # سيحتوي على Crash/Giveback
+                        exit_reason = f"Hydra_{hydra_res.get('reason','Unknown')}"
                         
-                # 🛡️ 2. Legacy Guardian (V2/V3)
+                # 🛡️ 2. Legacy (Exact Thresholds)
                 if not exit_reason and self.proc.guardian_legacy:
-                    # تمرير None لدفتر الطلبات (غير موجود في الباكتست)
+                    # حقن المتغيرات الدقيقة
+                    self.proc.guardian_legacy.configure_thresholds(
+                        v2_panic=legacy_v2, 
+                        v3_hard=legacy_v3, 
+                        v3_soft=legacy_v3-0.1, # مشتق منطقي
+                        v3_ultra=0.99
+                    )
                     legacy_res = self.proc.guardian_legacy.analyze_position(
                         snapshot['1m'], snapshot['5m'], snapshot['15m'], trade['entry_price'], order_book=None
                     )
                     if legacy_res['action'] == 'EXIT_HARD':
-                        exit_reason = "Legacy_Panic"
+                        exit_reason = f"Legacy_{legacy_res.get('reason','Unknown')}"
 
-                # 🛑 3. Hard TP/SL
                 if not exit_reason:
-                    if current_price >= trade['tp']: exit_reason = "TP_Hit"
-                    elif current_price <= trade['sl']: exit_reason = "SL_Hit"
+                    if current_price >= trade['tp']: exit_reason = "TP"
+                    elif current_price <= trade['sl']: exit_reason = "SL"
                 
-                # تنفيذ الخروج وتسجيل السبب
                 if exit_reason:
                     pnl_pct = (current_price - trade['entry_price']) / trade['entry_price']
                     pnl_usd = trade['size'] * pnl_pct
                     self.portfolio.capital += (trade['size'] + pnl_usd)
                     trades_log.append(pnl_pct)
                     
-                    # 📝 تسجيل من هو البطل؟
+                    # تسجيل دقيق للسبب
                     if 'Crash' in exit_reason: self.portfolio.guardian_log['hydra_crash'] += 1
                     elif 'Giveback' in exit_reason: self.portfolio.guardian_log['hydra_giveback'] += 1
-                    elif 'Legacy' in exit_reason: self.portfolio.guardian_log['legacy_panic'] += 1
-                    elif 'TP' in exit_reason: self.portfolio.guardian_log['tp_hit'] += 1
-                    elif 'SL' in exit_reason: self.portfolio.guardian_log['sl_hit'] += 1
+                    elif 'V2' in exit_reason: self.portfolio.guardian_log['legacy_v2'] += 1
+                    elif 'V3' in exit_reason: self.portfolio.guardian_log['legacy_v3'] += 1
+                    elif 'TP' in exit_reason: self.portfolio.guardian_log['tp'] += 1
+                    elif 'SL' in exit_reason: self.portfolio.guardian_log['sl'] += 1
+                    
+                    if pnl_usd > self.portfolio.stats['max_win_usd']: self.portfolio.stats['max_win_usd'] = pnl_usd
+                    if pnl_usd < self.portfolio.stats['max_loss_usd']: self.portfolio.stats['max_loss_usd'] = pnl_usd
                     
                     del self.portfolio.active_trades[sym]
             
@@ -281,82 +304,96 @@ class BacktestSimulator:
             current_idx += 1
         
         wins = len([p for p in trades_log if p > 0])
-        total = len(trades_log)
-        wr = (wins/total*100) if total > 0 else 0.0
+        losses = len(trades_log) - wins
+        wr = (wins/len(trades_log)*100) if len(trades_log) > 0 else 0.0
         
         return {
             'final_capital': self.portfolio.capital, 
             'win_rate': wr, 
-            'trades': total,
-            'guards': self.portfolio.guardian_log
+            'trades_count': len(trades_log),
+            'wins': wins,
+            'losses': losses,
+            'guards': self.portfolio.guardian_log,
+            'stats': self.portfolio.stats
         }
 
     # ==========================================================================
-    # 5. Grid Search Manager (توليد آلي)
+    # 5. Master Grid Search (Expanded)
     # ==========================================================================
     async def optimize_dna(self):
         best_dna = {}
-        regimes = ['RANGE'] # للسرعة
+        regimes = ['RANGE'] 
         
-        # 🎲 توليد الاحتمالات (Grid Search Generation)
-        # نولد أوزان بحيث يكون مجموعها 1.0 تقريباً
-        # Titan, Pattern, Sniper, MC (الثابت 0.1)
-        # سنغير Titan و Pattern و Sniper
-        
-        possible_weights = []
-        # توليد مبسط لتفادي الانفجار الحسابي
-        # (Titan, Patterns, Sniper)
-        combos = [
-            (0.3, 0.3, 0.3), # Balanced
-            (0.5, 0.2, 0.2), # Trend Focus
-            (0.2, 0.5, 0.2), # Pattern Focus
-            (0.2, 0.2, 0.5), # Sniper Focus
-            (0.4, 0.4, 0.1), # Tech Focus
+        # 1. متغيرات الأوزان
+        weight_opts = [
+            {'titan': 0.3, 'patterns': 0.3, 'sniper': 0.3, 'mc': 0.1}, # Balanced
+            {'titan': 0.5, 'patterns': 0.2, 'sniper': 0.2, 'mc': 0.1}, # Trend
         ]
         
-        possible_thresholds = [0.50, 0.60] # تجربتين للعتبة
+        # 2. متغيرات العتبات (Entry)
+        entry_thresh_opts = [0.55, 0.60]
+        
+        # 3. متغيرات Hydra (Crash/Giveback)
+        hydra_crash_opts = [0.60, 0.70] # هل نجزع عند 60% أم 70%؟
+        hydra_give_opts = [0.65, 0.75]
+        
+        # 4. متغيرات Legacy (V2/V3)
+        legacy_v2_opts = [0.95, 0.98] # V2 Panic
+        legacy_v3_opts = [0.95] # V3 Hard (نثبته لتقليل الاحتمالات قليلاً)
+        
+        # Grid Size: 2 * 2 * 2 * 2 * 2 * 1 = 32 Combination
         
         for regime in regimes:
-            print(f"\n🧪 Optimizing {regime} (Grid Search: {len(combos)*len(possible_thresholds)} combinations)...")
-            print("-" * 100)
-            print(f"{'WEIGHTS (Ti/Pat/Sn)':<25} | {'THR':<4} | {'CAPITAL':<9} | {'WR%':<5} | {'TRDS':<4} | {'GUARDS (Crsh/Gvbk/Leg/TP/SL)'}")
-            print("-" * 100)
+            print(f"\n🧪 Optimizing {regime} (Master Grid: 32 combos)...")
+            # رأس الجدول العريض جداً
+            header = f"{'W(Ti/Pat/Sn)':<15} | {'En-Th':<5} | {'H-Crsh':<6} | {'H-Giv':<5} | {'L-V2':<4} | {'CAPITAL':<9} | {'W/L':<7} | {'Guards(Cr/Gv/V2/V3)'}"
+            print("-" * len(header))
+            print(header)
+            print("-" * len(header))
             
             best_score = -9999
             best_config = None
             
-            for w in combos:
-                weights = {'titan': w[0], 'patterns': w[1], 'sniper': w[2], 'mc': 0.1}
-                for thresh in possible_thresholds:
-                    
-                    res = await self.run_simulation(regime, weights, thresh)
-                    
-                    g = res['guards']
-                    guard_str = f"{g['hydra_crash']}/{g['hydra_giveback']}/{g['legacy_panic']}/{g['tp_hit']}/{g['sl_hit']}"
-                    w_str = f"{w[0]}/{w[1]}/{w[2]}"
-                    
-                    print(f"{w_str:<25} | {thresh:<4} | ${res['final_capital']:.1f}   | {res['win_rate']:.1f}% | {res['trades']:<4} | {guard_str}")
-                    
-                    if res['final_capital'] > best_score:
-                        best_score = res['final_capital']
-                        best_config = {'w': weights, 'th': thresh, 'guards': g}
+            # حلقة التوليفات الكبرى
+            for w, e_th, h_c, h_g, l_v2, l_v3 in itertools.product(weight_opts, entry_thresh_opts, hydra_crash_opts, hydra_give_opts, legacy_v2_opts, legacy_v3_opts):
+                
+                res = await self.run_simulation(regime, w, e_th, h_c, h_g, l_v2, l_v3)
+                
+                w_str = f"{w['titan']}/{w['patterns']}/{w['sniper']}"
+                wl_str = f"{res['wins']}/{res['losses']}"
+                g = res['guards']
+                g_str = f"{g['hydra_crash']}/{g['hydra_giveback']}/{g['legacy_v2']}/{g['legacy_v3']}"
+                
+                print(f"{w_str:<15} | {e_th:<5} | {h_c:<6} | {h_g:<5} | {l_v2:<4} | ${res['final_capital']:.1f}   | {wl_str:<7} | {g_str}")
+                
+                if res['final_capital'] > best_score:
+                    best_score = res['final_capital']
+                    best_config = {'w': w, 'e_th': e_th, 'h_c': h_c, 'h_g': h_g, 'l_v2': l_v2, 'l_v3': l_v3, 'res': res}
             
             if best_config:
                 best_dna[regime] = {
                     "model_weights": best_config['w'],
                     "ob_settings": {"wall_ratio_limit": 0.4, "imbalance_thresh": 0.5},
-                    "filters": {"l1_min_score": best_config['th'] * 100, "l3_conf_thresh": 0.65} 
+                    "filters": {"l1_min_score": best_config['e_th'] * 100, "l3_conf_thresh": 0.65},
+                    # حفظ عتبات الحراس الدقيقة
+                    "guard_settings": {
+                        "hydra_crash": best_config['h_c'],
+                        "hydra_giveback": best_config['h_g'],
+                        "legacy_v2": best_config['l_v2'],
+                        "legacy_v3": best_config['l_v3']
+                    }
                 }
-                g = best_config['guards']
-                print("-" * 100)
-                print(f"🏆 WINNER ({regime}): Weights={best_config['w']} | Thresh={best_config['th']}")
-                print(f"🛡️ Guardian Report: Hydra Saved: {g['hydra_crash']+g['hydra_giveback']} | Legacy Saved: {g['legacy_panic']}")
-                print("=" * 100)
+                
+                s = best_config['res']['stats']
+                print("-" * len(header))
+                print(f"🏆 WINNER ({regime}): Profit=${(best_score-1000):.2f} | Max DD: {s['max_drawdown_pct']*100:.2f}%")
+                print(f"   ⚙️ Config: H-Crash={best_config['h_c']}, H-Give={best_config['h_g']}, V2={best_config['l_v2']}")
+                print("=" * len(header))
 
         return best_dna
 
 async def run_strategic_optimization_task():
-    print("\n🧪 [STRATEGIC BACKTEST V40.0] Full Grid & Guardian Telemetry...")
+    print("\n🧪 [STRATEGIC BACKTEST V42.1] Exact Threshold Grid...")
     from r2 import R2Service
     r2 = R2Service()
     dm = DataManager(None, None, r2)
@@ -376,6 +413,7 @@ async def run_strategic_optimization_task():
         if reg in hub.strategies: 
             hub.strategies[reg].model_weights.update(data['model_weights'])
             hub.strategies[reg].filters = data['filters']
+            # هنا سنحتاج لتحديث AdaptiveHub لاحقاً ليخزن guard_settings
             
     await hub._save_state_to_r2()
     await dm.close()