Update backtest_engine.py

backtest_engine.py

@@ -1,10 +1,8 @@
 # ============================================================
-# 🧪 backtest_engine.py (V43.0 - GEM-Architect: Disk-Swapped Grid)
+# 🧪 backtest_engine.py (V45.0 - GEM-Architect: Auto-Deploy)
 # ============================================================
-# التغييرات الجوهرية (Anti-Crash Architecture):
-# 1. التخزين المؤقت: حفظ البيانات في ملفات محلية بدلاً من RAM.
-# 2. قلب حلقة المعالجة: (Loop Coins -> Loop Strategies) لتقليل التحميل.
-# 3. استهلاك ذاكرة منخفض جداً (عملة واحدة في كل لحظة).
+# الميزة القاتلة: Find Best -> Save -> HOT RELOAD.
+# بمجرد انتهاء الباكتست، النظام يتبنى الإعدادات الجديدة فوراً.
 # ============================================================
 
 import asyncio
@@ -21,16 +19,29 @@ from ml_engine.processor import MLProcessor, SystemLimits
 from ml_engine.data_manager import DataManager
 from learning_hub.adaptive_hub import StrategyDNA
 
-# كتم الضوضاء
 logging.getLogger('ml_engine.patterns').setLevel(logging.WARNING)
-
-# مسار الكاش المؤقت
 CACHE_DIR = "backtest_cache_temp"
 
+class VirtualPortfolio:
+    def __init__(self, initial_capital=1000.0):
+        self.capital = initial_capital
+        self.active_trades = {} 
+        self.stats = {"max_win_usd": 0.0, "max_loss_usd": 0.0, "max_drawdown_pct": 0.0, "max_runup_pct": 0.0}
+        self.guardian_log = {'hydra_crash': 0, 'hydra_giveback': 0, 'legacy_v2': 0, 'legacy_v3': 0, 'tp': 0, 'sl': 0}
+        self.MAX_SLOTS_MAP = {'BULL': 6, 'BEAR': 3, 'RANGE': 5, 'DEAD': 2}
+    
+    def can_open_trade(self, regime):
+        max_slots = self.MAX_SLOTS_MAP.get(regime, 4)
+        return len(self.active_trades) < max_slots
+
+    def calculate_size(self, confidence, regime):
+        return self.capital * 0.10
+
 class BacktestSimulator:
     def __init__(self, data_manager, processor):
         self.dm = data_manager
         self.proc = processor
+        self.history_cache = {} 
         self.DAYS_TO_FETCH = 7 
         self.CHUNK_LIMIT = 1000
         
@@ -41,27 +52,19 @@ class BacktestSimulator:
             'XLM/USDT', 'TRX/USDT', 'LTC/USDT'
         ]
         
-        # إنشاء مجلد مؤقت
         if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        
-        print("🧪 [Backtest Engine V43.0] Disk-Swap Memory Protection Active.")
+        print("🧪 [Backtest Engine V45.0] Auto-Deploy Grid Initialized.")
 
-    # ==========================================================================
-    # 1. Data Loading (Download -> Save to Disk -> Clear RAM)
-    # ==========================================================================
     async def fetch_deep_history_1m(self):
-        print(f"\n⏳ [Data] Downloading {len(self.TARGET_COINS)} coins to Disk Cache...")
+        print(f"\n⏳ [Data] Downloading {len(self.TARGET_COINS)} coins to Disk...")
         end_time_ms = int(time.time() * 1000)
         start_time_ms = end_time_ms - (self.DAYS_TO_FETCH * 24 * 60 * 60 * 1000)
         
         for sym in self.TARGET_COINS:
             safe_sym = sym.replace('/', '_')
             file_path = f"{CACHE_DIR}/{safe_sym}.pkl"
-            
-            # إذا كان الملف موجوداً وحديثاً، نتجاوز التحميل (تسريع)
             if os.path.exists(file_path):
-                print(f"   📂 {sym:<10} [Cached]", end="", flush=True)
-                print(f" ✅")
+                print(f"   📂 {sym:<10} [Cached] ✅")
                 continue
 
             print(f"   ⬇️  {sym:<10}", end="", flush=True) 
@@ -86,21 +89,13 @@ class BacktestSimulator:
                 df['datetime'] = pd.to_datetime(df['timestamp'], unit='ms')
                 df = df.set_index('datetime').sort_index()
                 
-                # 🔥 الحفظ على القرص وتفريغ الذاكرة
                 df.to_pickle(file_path)
                 print(f" ✅ Saved ({len(df)})")
-                
-                # تنظيف المتغيرات
-                del df
-                del all_candles
+                del df, all_candles
             else:
                 print(" ⚠️ No Data")
-        
-        print(f"✅ Download Complete. RAM is clear.")
+        print(f"✅ Download Complete.")
 
-    # ==========================================================================
-    # 2. Snapshot Helper
-    # ==========================================================================
     def get_market_snapshot(self, df_full, end_idx):
         try:
             LOOKBACK_WINDOW = 6000
@@ -116,102 +111,25 @@ class BacktestSimulator:
                 resampled = slice_1m.resample(rule).agg(agg).dropna()
                 if len(resampled) < 20: return None
                 timeframes[tf] = resampled[cols_order].values.tolist()
-                
             return timeframes
         except: return None
 
-    # ==========================================================================
-    # 3. Process Logic
-    # ==========================================================================
-    async def process_market_layers(self, snapshot, current_price, weights, l1_threshold):
-        # L2
-        titan_score = 0.5
-        if self.proc.titan:
-            res = await asyncio.to_thread(self.proc.titan.predict, snapshot)
-            titan_score = res.get('score', 0.5)
-
-        pattern_score = 0.5
-        if self.proc.pattern_engine:
-            self.proc.pattern_engine.configure_thresholds(weights=SystemLimits.PATTERN_TF_WEIGHTS, bull_thresh=0.5, bear_thresh=0.4)
-            res = await self.proc.pattern_engine.detect_chart_patterns(snapshot)
-            pattern_score = res.get('pattern_confidence', 0.5)
-
-        mc_light = 0.5
-        if self.proc.mc_analyzer:
-            closes = [c[4] for c in snapshot['1h']]
-            raw = self.proc.mc_analyzer.run_light_check(closes)
-            mc_light = 0.5 + (raw * 5.0)
-
-        w_t = weights['titan']; w_p = weights['patterns']; w_m = weights['mc']
-        total_w = w_t + w_p + w_m
-        l2_score = ((titan_score * w_t) + (pattern_score * w_p) + (mc_light * w_m)) / total_w
-
-        if l2_score < l1_threshold: return None 
-
-        # Oracle
-        mc_adv_score = 0.0
-        if self.proc.mc_analyzer:
-            closes = [c[4] for c in snapshot['1h']]
-            mc_adv_score = self.proc.mc_analyzer.run_advanced_simulation(closes)
-        l3_score = l2_score + mc_adv_score
-
-        oracle_decision = {'action': 'WAIT'}
-        if self.proc.oracle:
-            self.proc.oracle.set_threshold(0.60) 
-            oracle_input = {'ohlcv': snapshot, 'current_price': current_price, 'titan_score': titan_score, 'mc_score': mc_light, 'patterns_score': pattern_score}
-            oracle_decision = await self.proc.oracle.predict(oracle_input)
-
-        if oracle_decision['action'] not in ['BUY', 'WATCH']: return None
-
-        # Sniper
-        sniper_res = {'signal': 'WAIT'}
-        if self.proc.sniper:
-            self.proc.sniper.configure_settings(threshold=0.40, wall_ratio=0.9, w_ml=1.0, w_ob=0.0)
-            sniper_res = await self.proc.sniper.check_entry_signal_async(snapshot['1m'], None)
-
-        if sniper_res['signal'] == 'BUY':
-            return {
-                'entry_price': sniper_res.get('entry_price', current_price),
-                'tp': oracle_decision.get('primary_tp'),
-                'sl': oracle_decision.get('sl_price'),
-                'oracle_conf': oracle_decision.get('confidence'),
-                'l2_score': l3_score
-            }
-        return None
-
-    # ==========================================================================
-    # 4. Inverted Simulation Loop (Fast & Low Memory)
-    # ==========================================================================
     async def run_single_coin_sim(self, symbol, df_history, combinations):
-        """
-        تقوم بتشغيل جميع التوليفات (Combinations) على عملة واحدة دفعة واحدة.
-        """
-        # نتائج هذه العملة لكل توليفة
-        # Key: Combo_Index, Value: List of Trades
         coin_results = {i: [] for i in range(len(combinations))}
-        
         full_index = df_history.index
         start_idx = 6000
         end_idx = len(full_index) - 1
         current_idx = start_idx
         
-        # لتقليل العمليات، نحسب المؤشرات مرة واحدة
-        # ولكن بما أن الأوزان تغير L2 Score، يجب إعادة الحساب جزئياً
-        # الحل: نحسب النماذج (Titan, Pattern) مرة واحدة لكل شمعة، ثم نطبق الأوزان المختلفة
-        
         while current_idx < end_idx:
-            # كل 15 دقيقة
-            current_time = full_index[current_idx]
-            
-            # --- 1. حساب القيم الخام للشمعة الحالية (مرة واحدة) ---
             snapshot = self.get_market_snapshot(df_history, current_idx)
             if not snapshot: 
-                current_idx += 15
+                current_idx += 30
                 continue
-                
+            
             current_price = snapshot['1m'][-1][4]
             
-            # حساب قيم النماذج (ثقيلة)
+            # حساب النماذج مرة واحدة
             titan_s = 0.5
             if self.proc.titan: titan_s = (await asyncio.to_thread(self.proc.titan.predict, snapshot)).get('score', 0.5)
             
@@ -224,55 +142,67 @@ class BacktestSimulator:
             if self.proc.mc_analyzer:
                 mc_s = 0.5 + (self.proc.mc_analyzer.run_light_check([c[4] for c in snapshot['1h']]) * 5.0)
 
-            # --- 2. تجربة جميع التوليفات على هذه القيم الجاهزة (سريع جداً) ---
-            for i, config in enumerate(combinations):
-                w = config['w']
-                l1_th = config['e_th']
-                
-                # حساب L2
-                l2 = ((titan_s * w['titan']) + (patt_s * w['patterns']) + (mc_s * w['mc'])) / (w['titan']+w['patterns']+w['mc'])
-                
-                if l2 < l1_th: continue # فشل
-                
-                # Oracle & Sniper (نحسبهم مرة واحدة إذا نجح L2 لأي توليفة، لكن للتبسيط نحسبهم هنا)
-                # بما أن Oracle/Sniper لا يعتمدون على الأوزان بشكل مباشر في قرارهم (بل يأخذونها كمدخلات)
-                # سنقوم بتشغيلهم فقط إذا نجح L2
-                
-                # Oracle
+            oracle_ok = False
+            sniper_ok = False
+            
+            oracle_dec = {'action': 'WAIT'}
+            if self.proc.oracle:
+                self.proc.oracle.set_threshold(0.50) 
                 oracle_dec = await self.proc.oracle.predict({
                     'ohlcv': snapshot, 'current_price': current_price,
                     'titan_score': titan_s, 'mc_score': mc_s, 'patterns_score': patt_s
                 })
-                if oracle_dec['action'] not in ['BUY', 'WATCH']: continue
+                if oracle_dec['action'] in ['BUY', 'WATCH']: oracle_ok = True
                 
-                # Sniper
+            if oracle_ok and self.proc.sniper:
+                self.proc.sniper.configure_settings(threshold=0.35, wall_ratio=0.9, w_ml=1.0, w_ob=0.0)
                 sniper_res = await self.proc.sniper.check_entry_signal_async(snapshot['1m'], None)
-                if sniper_res['signal'] != 'BUY': continue
-                
-                # ✅ تسجيل نجاح صفقة لهذه التوليفة
-                # محاكاة النتيجة (بعد ساعتين)
+                if sniper_res['signal'] == 'BUY': sniper_ok = True
+
+            if oracle_ok and sniper_ok:
                 future_idx = min(current_idx + 120, len(df_history)-1)
-                exit_price = df_history.iloc[future_idx]['close']
-                pnl = (exit_price - current_price) / current_price
-                
-                # فحص الحراس (بناءً على config)
-                # هنا نختصر: نفترض الخروج بعد ساعتين أو TP/SL افتراضي
-                # لتسريع الـ Grid الضخم
+                price_path = df_history.iloc[current_idx:future_idx]['close'].values
                 
-                coin_results[i].append(pnl)
-                
-            current_idx += 30 # قفزة 30 دقيقة
+                # قيم الحراس
+                hydra_probs = {'crash': 0.0, 'giveback': 0.0}
+                if self.proc.guardian_hydra:
+                    h_res = self.proc.guardian_hydra.analyze_position(symbol, snapshot['1m'], snapshot['5m'], snapshot['15m'], {})
+                    hydra_probs = h_res.get('probs', {'crash': 0.0, 'giveback': 0.0})
+
+                legacy_scores = {'v2': 0.0, 'v3': 0.0}
+                if self.proc.guardian_legacy:
+                    l_res = self.proc.guardian_legacy.analyze_position(snapshot['1m'], snapshot['5m'], snapshot['15m'], current_price)
+                    legacy_scores = l_res.get('scores', {'v2': 0.0, 'v3': 0.0})
+
+                for i, config in enumerate(combinations):
+                    w = config['w']
+                    l1_th = config['e_th']
+                    
+                    l2 = ((titan_s * w['titan']) + (patt_s * w['patterns']) + (mc_s * w['mc'])) / (w['titan']+w['patterns']+w['mc'])
+                    if l2 < l1_th: continue 
+                    
+                    entry_p = current_price
+                    exit_p = price_path[-1] 
+                    
+                    if hydra_probs['crash'] >= config['h_c']: 
+                        exit_p = price_path[len(price_path)//3] 
+                    elif hydra_probs['giveback'] >= config['h_g']:
+                        exit_p = price_path[len(price_path)//2]
+                    elif legacy_scores['v2'] >= config['l_v2']:
+                        exit_p = price_path[len(price_path)//3]
+
+                    pnl = (exit_p - entry_p) / entry_p
+                    coin_results[i].append(pnl)
+
+            current_idx += 30 
             
         return coin_results
 
-    # ==========================================================================
-    # 5. Master Grid Search (Optimized Memory)
-    # ==========================================================================
     async def optimize_dna(self):
         best_dna = {}
         regimes = ['RANGE'] 
         
-        # إعداد التوليفات
+        # --- The Grand Grid ---
         weight_opts = [
             {'titan': 0.3, 'patterns': 0.3, 'sniper': 0.3, 'mc': 0.1}, 
             {'titan': 0.5, 'patterns': 0.2, 'sniper': 0.2, 'mc': 0.1},
@@ -284,7 +214,6 @@ class BacktestSimulator:
         legacy_v2_opts = [0.95, 0.98]
         legacy_v3_opts = [0.95]
 
-        # تجميع كل الاحتمالات في قائمة واحدة
         combinations = []
         for w, e, hc, hg, l2, l3 in itertools.product(
             weight_opts, entry_thresh_opts, hydra_crash_opts, hydra_give_opts, legacy_v2_opts, legacy_v3_opts
@@ -294,37 +223,28 @@ class BacktestSimulator:
                 'total_pnl_usd': 0.0, 'trades': 0, 'wins': 0
             })
         
-        print(f"\n🧪 Testing {len(combinations)} Strategies on 20 Coins (Disk-Swapped)...")
+        print(f"\n🧪 Testing {len(combinations)} Strategies on Disk-Swap...")
         
-        # 🔥 حلقة العملات (تحمل وتحذف واحدة تلو الأخرى)
         for sym in self.TARGET_COINS:
             safe_sym = sym.replace('/', '_')
             file_path = f"{CACHE_DIR}/{safe_sym}.pkl"
-            
             if not os.path.exists(file_path): continue
             
-            # 1. تحميل للذاكرة
-            print(f"   👉 Processing {sym}...", end="", flush=True)
+            print(f"   👉 {sym}...", end="", flush=True)
             df_history = pd.read_pickle(file_path)
             
-            # 2. تشغيل المحاكاة السريعة
             results = await self.run_single_coin_sim(sym, df_history, combinations)
             
-            # 3. تجميع النتائج
             for i, trades in results.items():
                 for pnl in trades:
-                    # تحديث المحفظة الافتراضية لكل استراتيجية
-                    # نفترض حجم صفقة 100$
                     profit_usd = 100.0 * pnl
                     combinations[i]['total_pnl_usd'] += profit_usd
                     combinations[i]['trades'] += 1
                     if pnl > 0: combinations[i]['wins'] += 1
             
-            # 4. تفريغ الذاكرة
             del df_history
             print(" Done.")
 
-        # 🔥 العثور على الفائز
         best_combo = sorted(combinations, key=lambda x: x['total_pnl_usd'], reverse=True)[0]
         
         regime = 'RANGE'
@@ -332,6 +252,7 @@ class BacktestSimulator:
             "model_weights": best_combo['w'],
             "ob_settings": {"wall_ratio_limit": 0.4, "imbalance_thresh": 0.5},
             "filters": {"l1_min_score": best_combo['e_th'] * 100, "l3_conf_thresh": 0.65},
+            # 🔥 الحفظ الجديد في DNA
             "guard_settings": {
                 "hydra_crash": best_combo['h_c'], "hydra_giveback": best_combo['h_g'],
                 "legacy_v2": best_combo['l_v2'], "legacy_v3": best_combo['l_v3']
@@ -341,18 +262,18 @@ class BacktestSimulator:
         print("-" * 100)
         print(f"🏆 GRAND WINNER ({regime}):")
         print(f"   💰 Total Profit: ${best_combo['total_pnl_usd']:.2f}")
-        print(f"   📊 Trades: {best_combo['trades']} (Win Rate: {(best_combo['wins']/best_combo['trades']*100 if best_combo['trades']>0 else 0):.1f}%)")
+        print(f"   📊 Trades: {best_combo['trades']} (WR: {(best_combo['wins']/best_combo['trades']*100 if best_combo['trades']>0 else 0):.1f}%)")
         print(f"   ⚙️ Config: {best_combo['w']} | Thresh: {best_combo['e_th']}")
+        print(f"   🛡️ Guards: Hydra(C:{best_combo['h_c']}/G:{best_combo['h_g']}) | Legacy(V2:{best_combo['l_v2']})")
         print("=" * 100)
         
-        # تنظيف الكاش
         try: shutil.rmtree(CACHE_DIR)
         except: pass
         
         return best_dna
 
 async def run_strategic_optimization_task():
-    print("\n🧪 [STRATEGIC BACKTEST V43.0] Disk-Swapped Grid...")
+    print("\n🧪 [STRATEGIC BACKTEST V45.0] Auto-Deploy Grid...")
     from r2 import R2Service
     r2 = R2Service()
     dm = DataManager(None, None, r2)
@@ -363,19 +284,33 @@ async def run_strategic_optimization_task():
     sim = BacktestSimulator(dm, proc)
     await sim.fetch_deep_history_1m()
     
+    # 1. العثور على الأفضل
     optimized_strategies = await sim.optimize_dna()
     
+    # 2. الحفظ في R2
     from learning_hub.adaptive_hub import AdaptiveHub
     hub = AdaptiveHub(r2)
     await hub.initialize()
+    
     for reg, data in optimized_strategies.items():
         if reg in hub.strategies: 
+            # تحديث الـ DNA
             hub.strategies[reg].model_weights.update(data['model_weights'])
             hub.strategies[reg].filters = data['filters']
+            # إضافة guard_settings يدوياً للكائن لأننا لم نعدل الكلاس
+            hub.strategies[reg].guard_settings = data['guard_settings']
             
     await hub._save_state_to_r2()
+    
+    # 3. 🔥 التفعيل الفوري (HOT RELOAD)
+    print("🚀 [Hot Reload] Applying new DNA to Live System...")
+    hub._inject_current_parameters()
+    
+    # إذا كان هناك دالة في processor لتحديث الحراس مباشرة، نستدعيها
+    # بما أننا نعتمد على SystemLimits، فإن الاستدعاء القادم للحراس سيقرأ القيم الجديدة تلقائياً
+    
     await dm.close()
-    print("✅ [STRATEGIC BACKTEST] Completed.")
+    print("✅ [STRATEGIC BACKTEST] Completed & ACTIVATED.")
 
 if __name__ == "__main__":
     asyncio.run(run_strategic_optimization_task())