Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V76.0 - GEM-Architect: Aggressive Force)
+# 🧪 backtest_engine.py (V81.0 - GEM-Architect: The Time Lord)
 # ============================================================
 
 import asyncio
@@ -11,6 +11,7 @@ import itertools
 import os
 import gc 
 import concurrent.futures
+from datetime import datetime, timezone
 from typing import Dict, Any, List
 
 try:
@@ -28,17 +29,35 @@ class HeavyDutyBacktester:
     def __init__(self, data_manager, processor):
         self.dm = data_manager
         self.proc = processor
-        self.GRID_DENSITY = 10 # كثافة عالية للدقة
-        self.BACKTEST_DAYS = 30 # شهر كامل
+        self.GRID_DENSITY = 10 # كثافة متوازنة للسرعة والدقة
         
         # 💰 إعدادات التوأم الرقمي
         self.INITIAL_CAPITAL = 10.0
         self.TRADING_FEES = 0.001 
         self.MAX_SLOTS = 4         
         
-        self.TARGET_COINS = [ 'SOL/USDT', 'XRP/USDT', 'DOGE/USDT', 'ADA/USDT', 'AVAX/USDT', 'LINK/USDT', 'TON/USDT', 'INJ/USDT', 'APT/USDT', 'OP/USDT', 'ARB/USDT', 'SUI/USDT', 'SEI/USDT', 'TIA/USDT', 'MATIC/USDT', 'PEPE/USDT', 'NEAR/USDT', 'RUNE/USDT', 'PYTH/USDT', 'WIF/USDT' ]
+        self.TARGET_COINS = [ 
+            'SOL/USDT', 'XRP/USDT', 'DOGE/USDT', 'ADA/USDT', 'AVAX/USDT', 
+            'LINK/USDT', 'TON/USDT', 'INJ/USDT', 'APT/USDT', 'OP/USDT', 
+            'ARB/USDT', 'SUI/USDT', 'SEI/USDT', 'TIA/USDT', 'BEST/USDT', 
+            'NEAR/USDT', 'RUNE/USDT', 'PYTH/USDT', 'WIF/USDT', 'PEPE/USDT', 
+            'SHIB/USDT', 'TRX/USDT', 'DOT/USDT', 'UNI/USDT', 'ONDO/USDT', 
+            'ENA/USDT', 'HBAR/USDT', 'XLM/USDT', 'TAO/USDT', 'ZK/USDT', 
+            'ZRO/USDT', 'KCS/USDT', 'ICP/USDT', 'SAND/USDT', 'AXS/USDT', 
+            'APE/USDT', 'GMT/USDT', 'CHZ/USDT', 'CFX/USDT', 'LDO/USDT', 
+            'FET/USDT', 'JTO/USDT', 'STRK/USDT', 'BLUR/USDT', 'ALT/USDT', 
+            'JUP/USDT', 'PENDLE/USDT', 'ETHFI/USDT', 'MEME/USDT', 'ATOM/USDT' 
+        ]
+        
+        self.force_start_date = None 
+        self.force_end_date = None   
+        
         if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        print(f"🧪 [Backtest V76.0] Aggressive Force Mode (Max Thresh 0.55).")
+        print(f"🧪 [Backtest V81.0] Time Lord Mode Ready.")
+
+    def set_date_range(self, start_str, end_str):
+        self.force_start_date = start_str
+        self.force_end_date = end_str
 
     # ==============================================================
     # 🛠️ Helpers
@@ -52,15 +71,15 @@ class HeavyDutyBacktester:
     # ==============================================================
     async def _process_single_coin_task(self, sym, start_time_ms, end_time_ms):
         safe_sym = sym.replace('/', '_')
-        scores_file = f"{CACHE_DIR}/{safe_sym}_fullstack_scores.pkl"
+        period_suffix = f"{start_time_ms}_{end_time_ms}"
+        scores_file = f"{CACHE_DIR}/{safe_sym}_{period_suffix}_scores.pkl"
         
         if os.path.exists(scores_file):
-            print(f"   📂 {sym} scores ready. Skipping.")
+            # print(f"   📂 {sym} ready.")
             return True
 
         print(f"   ⚙️ Simulating {sym}...", end="", flush=True)
         
-        # تهيئة المتغيرات لتجنب أخطاء النطاق
         all_candles_1m = []
         df_1m = None
         frames = {}
@@ -184,27 +203,31 @@ class HeavyDutyBacktester:
     # PHASE 1: Main Loop
     # ==============================================================
     async def generate_truth_data(self):
-        print(f"\n🚜 [Phase 1] Processing Logic Tree + Titan ({self.BACKTEST_DAYS} Days)...")
-        end_time_ms = int(time.time() * 1000)
-        start_time_ms = end_time_ms - (self.BACKTEST_DAYS * 24 * 60 * 60 * 1000) 
+        if self.force_start_date and self.force_end_date:
+            dt_start = datetime.strptime(self.force_start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+            dt_end = datetime.strptime(self.force_end_date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+            start_time_ms = int(dt_start.timestamp() * 1000)
+            end_time_ms = int(dt_end.timestamp() * 1000)
+            print(f"\n🚜 [Phase 1] Processing Era: {self.force_start_date} -> {self.force_end_date}")
+        else:
+            return 
         
         chunk_size = 4
         chunks = [self.TARGET_COINS[i:i + chunk_size] for i in range(0, len(self.TARGET_COINS), chunk_size)]
         
         for chunk_idx, chunk in enumerate(chunks):
-            print(f"\n--- 📦 Processing Batch {chunk_idx + 1}/{len(chunks)} ---")
+            # print(f"--- Processing Batch {chunk_idx + 1}/{len(chunks)} ---")
             for sym in chunk:
                 try:
                     await asyncio.wait_for(
                         self._process_single_coin_task(sym, start_time_ms, end_time_ms),
-                        timeout=180.0 
+                        timeout=300.0 
                     )
                 except asyncio.TimeoutError:
-                    print(f" 💀 [WATCHDOG] Killed {sym}. Moving on...")
+                    print(f" 💀 Killed {sym}. Moving on...")
                     gc.collect()
-            
             gc.collect()
-            await asyncio.sleep(2.0)
+            await asyncio.sleep(1.0)
 
     # ==============================================================
     # PHASE 2: Portfolio Digital Twin Engine
@@ -234,7 +257,7 @@ class HeavyDutyBacktester:
             entry_thresh = config['thresh']
             
             for ts, group in grouped_by_time:
-                # 1. Exit Logic
+                # Exit
                 active_symbols = list(wallet["positions"].keys())
                 current_prices = {row['symbol']: row['close'] for _, row in group.iterrows()}
                 
@@ -254,7 +277,7 @@ class HeavyDutyBacktester:
                             del wallet["positions"][sym]
                             wallet["trades_history"].append({'pnl': net_pnl})
 
-                # 2. Entry Logic
+                # Entry
                 if len(wallet["positions"]) < max_slots:
                     free_capital = wallet["balance"] - wallet["allocated"]
                     slots_left = max_slots - len(wallet["positions"])
@@ -287,116 +310,88 @@ class HeavyDutyBacktester:
                 
                 if wallet["balance"] < 1.0 and len(wallet["positions"]) == 0: break
 
-            # Analytics
             trades = wallet["trades_history"]
             if trades:
                 net_profit = wallet["balance"] - initial_capital
                 pnls = [t['pnl'] for t in trades]
                 wins = [p for p in pnls if p > 0]
                 losses = [p for p in pnls if p <= 0]
+                win_rate = (len(wins) / len(trades)) * 100
+                max_dd = 0 # Simplified for speed
                 
-                win_count = len(wins); loss_count = len(losses)
-                total_trades = len(trades)
-                win_rate = (win_count / total_trades) * 100 if total_trades > 0 else 0
-                max_single_win = max(pnls) if pnls else 0.0
-                max_single_loss = min(pnls) if pnls else 0.0
-                
-                current_win_streak = 0; max_win_streak = 0
-                current_loss_streak = 0; max_loss_streak = 0
-                for p in pnls:
-                    if p > 0:
-                        current_win_streak += 1; current_loss_streak = 0
-                        if current_win_streak > max_win_streak: max_win_streak = current_win_streak
-                    else:
-                        current_loss_streak += 1; current_win_streak = 0
-                        if current_loss_streak > max_loss_streak: max_loss_streak = current_loss_streak
-
                 results.append({
                     'config': config,
                     'final_balance': wallet["balance"],
                     'net_profit': net_profit,
-                    'total_trades': total_trades,
-                    'win_count': win_count, 'loss_count': loss_count,
+                    'total_trades': len(trades),
                     'win_rate': win_rate,
-                    'max_single_win': max_single_win, 'max_single_loss': max_single_loss,
-                    'max_win_streak': max_win_streak, 'max_loss_streak': max_loss_streak
+                    'max_drawdown': max_dd
                 })
             else:
-                 results.append({
-                     'config': config, 'final_balance': initial_capital, 'net_profit': 0.0, 
-                     'total_trades': 0, 'win_count': 0, 'loss_count': 0, 'win_rate': 0.0,
-                     'max_single_win': 0.0, 'max_single_loss': 0.0, 'max_win_streak': 0, 'max_loss_streak': 0
-                 })
+                 results.append({'config': config, 'final_balance': initial_capital, 'net_profit': 0.0, 'total_trades': 0, 'win_rate': 0.0, 'max_drawdown': 0})
 
         return results
 
-    async def run_optimization(self):
-        # Ensure directory
-        if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        
+    async def run_optimization(self, target_regime="RANGE"):
         await self.generate_truth_data()
         
-        score_files = [os.path.join(CACHE_DIR, f) for f in os.listdir(CACHE_DIR) if f.endswith(f'_fullstack_scores.pkl')]
-        if not score_files:
-            print("❌ No Full-Stack signals found.")
+        # فلترة الملفات حسب الفترة الزمنية الحالية فقط
+        # نستخدم التواريخ في اسم الملف لضمان العزل
+        start_ts = int(datetime.strptime(self.force_start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc).timestamp() * 1000)
+        end_ts = int(datetime.strptime(self.force_end_date, "%Y-%m-%d").replace(tzinfo=timezone.utc).timestamp() * 1000)
+        period_id = f"{start_ts}_{end_ts}"
+        
+        current_period_files = []
+        for f in os.listdir(CACHE_DIR):
+            if f.endswith('_scores.pkl') and period_id in f:
+                current_period_files.append(os.path.join(CACHE_DIR, f))
+
+        if not current_period_files:
+            print(f"❌ No signals for {target_regime} ({self.force_start_date}).")
             return None
 
-        print(f"\n🧩 [Phase 2] Running Aggressive Force Simulation...")
+        print(f"\n🧩 [Phase 2] Optimizing for {target_regime}...")
         print(f"   💰 Start Capital: ${self.INITIAL_CAPITAL}")
         
         w_titan_range = np.linspace(0.4, 0.9, num=self.GRID_DENSITY)
         w_struct_range = np.linspace(0.1, 0.6, num=self.GRID_DENSITY)
-        
-        # 🔥🔥🔥 The Force: Forced Aggression (0.20 to 0.55) 🔥🔥🔥
-        thresh_range = np.linspace(0.20, 0.55, num=self.GRID_DENSITY)
+        # نستخدم طيف واسع جداً (0.20 - 0.85) لنغطي كل الحالات
+        thresh_range = np.linspace(0.20, 0.85, num=self.GRID_DENSITY)
         
         combinations = []
         for wt, ws, th in itertools.product(w_titan_range, w_struct_range, thresh_range):
             if 0.9 <= (wt + ws) <= 1.1:
                 combinations.append({'w_titan': round(wt, 2), 'w_struct': round(ws, 2), 'thresh': round(th, 2)})
             
-        print(f"   📊 Simulating {len(combinations):,} scenarios...")
-        
         final_results = []
         batch_size = max(20, len(combinations) // (os.cpu_count() * 2))
         batches = [combinations[i:i+batch_size] for i in range(0, len(combinations), batch_size)]
         
         with concurrent.futures.ProcessPoolExecutor() as executor:
-            futures = [executor.submit(self._worker_optimize, batch, score_files, 
+            futures = [executor.submit(self._worker_optimize, batch, current_period_files, 
                                        self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS) 
                        for batch in batches]
             for future in concurrent.futures.as_completed(futures):
                 try: final_results.extend(future.result())
                 except Exception as e: print(f"Grid Error: {e}")
 
-        if not final_results:
-            print("⚠️ No profitable config found.")
-            return None
+        if not final_results: return None
             
         best = sorted(final_results, key=lambda x: x['final_balance'], reverse=True)[0]
         
         print("\n" + "="*60)
-        print(f"🏆 CHAMPION AGGRESSIVE REPORT ({self.BACKTEST_DAYS} Days):")
-        print(f"   💰 Final Balance:   ${best['final_balance']:,.2f}")
-        print(f"   🚀 Net PnL:         ${best['net_profit']:,.2f}")
-        print("-" * 60)
-        print(f"   📊 Total Trades:    {best['total_trades']}")
-        print(f"   ✅ Winning Trades:  {best['win_count']}")
-        print(f"   ❌ Losing Trades:   {best['loss_count']}")
-        print(f"   📈 Win Rate:        {best['win_rate']:.1f}%")
-        print("-" * 60)
-        print(f"   🟢 Max Single Win:  ${best['max_single_win']:.2f}")
-        print(f"   🔴 Max Single Loss: ${best['max_single_loss']:.2f}")
-        print(f"   🔥 Max Win Streak:  {best['max_win_streak']} trades")
-        print(f"   🧊 Max Loss Streak: {best['max_loss_streak']} trades")
-        print("-" * 60)
+        print(f"🏆 CHAMPION REPORT [{target_regime}]:")
+        print(f"   📅 Period: {self.force_start_date} -> {self.force_end_date}")
+        print(f"   💰 Final Balance: ${best['final_balance']:,.2f}")
+        print(f"   🚀 Net PnL:       ${best['net_profit']:,.2f}")
+        print(f"   📈 Win Rate:      {best['win_rate']:.1f}%")
         print(f"   ⚙️ Config: Titan={best['config']['w_titan']} | Struct={best['config']['w_struct']} | Thresh={best['config']['thresh']}")
         print("="*60)
         
         return best['config']
 
 async def run_strategic_optimization_task():
-    print("\n🧪 [STRATEGIC BACKTEST] Starting Aggressive Force Optimization...")
+    print("\n🧪 [STRATEGIC BACKTEST] Time Lord Initiated...")
     r2 = R2Service()
     dm = DataManager(None, None, r2)
     proc = MLProcessor(dm)
@@ -405,25 +400,51 @@ async def run_strategic_optimization_task():
     await proc.initialize() 
     
     try:
+        hub = AdaptiveHub(r2)
+        await hub.initialize()
+        
+        # 🔥🔥🔥 خريطة السيناريوهات الشاملة (The Master Plan) 🔥🔥🔥
+        # التواريخ التي حددتها أنت بدقة
+        scenarios = [
+            # 1. BULL SCENARIOS (أقوى موجات الصعود)
+            # سنستخدم "موجة اعتماد ETF" كمعيار للبول رن
+            {"regime": "BULL", "start": "2024-01-01", "end": "2024-03-30"},
+            
+            # 2. BEAR SCENARIOS (أقوى موجات الهبوط)
+            # سنستخدم "التصحيح العنيف" كمعيار للهبوط
+            {"regime": "BEAR", "start": "2023-08-01", "end": "2023-09-15"},
+            
+            # 3. DEAD SCENARIOS (الركود القاتل)
+            # سنستخدم "يونيو 2023" كمعيار للموت السريري
+            {"regime": "DEAD", "start": "2023-06-01", "end": "2023-08-01"},
+            
+            # 4. RANGE SCENARIOS (التذبذب الممل)
+            # سنستخدم "يوليو 2024" كمعيار للعرضية
+            {"regime": "RANGE", "start": "2024-07-01", "end": "2024-09-30"}
+        ]
+        
         optimizer = HeavyDutyBacktester(dm, proc)
-        best_config = await optimizer.run_optimization()
         
-        if best_config:
-            from learning_hub.adaptive_hub import AdaptiveHub
-            hub = AdaptiveHub(r2)
-            await hub.initialize()
+        for scen in scenarios:
+            target = scen["regime"]
+            optimizer.set_date_range(scen["start"], scen["end"])
             
-            target_regime = "RANGE"
-            if target_regime in hub.strategies:
-                print(f"💉 Injecting DNA into {target_regime}...")
-                st = hub.strategies[target_regime]
+            # تشغيل التحسين لهذه الحقبة
+            best_config = await optimizer.run_optimization(target_regime=target)
+            
+            # حفظ النتائج فوراً في الذاكرة
+            if best_config and target in hub.strategies:
+                print(f"💉 Injecting Optimized DNA into {target}...")
+                st = hub.strategies[target]
                 st.model_weights['titan'] = best_config['w_titan']
                 st.model_weights['structure'] = best_config['w_struct']
                 st.filters['l1_min_score'] = best_config['thresh']
+        
+        # بعد انتهاء كل السيناريوهات، نحفظ الملف النهائي
+        await hub._save_state_to_r2()
+        hub._inject_current_parameters() # تفعيل الحالة الحالية
+        print(f"✅ [System] ALL DNA Updated & Saved Successfully.")
                 
-                await hub._save_state_to_r2()
-                hub._inject_current_parameters()
-                print(f"✅ [System] DNA Updated Successfully.")
     finally:
         await dm.close()