Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V69.0 - GEM-Architect: Detailed Analytics)
+# 🧪 backtest_engine.py (V71.1 - GEM-Architect: 6-Frame Turbo)
 # ============================================================
 
 import asyncio
@@ -27,48 +27,42 @@ class HeavyDutyBacktester:
     def __init__(self, data_manager, processor):
         self.dm = data_manager
         self.proc = processor
-        self.GRID_DENSITY = 15
-        self.BACKTEST_DAYS = 7 # 7 أيام كافية للتحسين السريع، 14 للدقة العالية
+        self.GRID_DENSITY = 10 
+        self.BACKTEST_DAYS = 7 
         
         # 💰 إعدادات التوأم الرقمي
         self.INITIAL_CAPITAL = 10.0
         self.TRADING_FEES = 0.001 
         self.MAX_SLOTS = 4         
         
-        self.TARGET_COINS = [ 'SOL/USDT', 'XRP/USDT', 'DOGE/USDT', 'ADA/USDT', 'AVAX/USDT', 'LINK/USDT', 'TON/USDT', 'INJ/USDT', 'APT/USDT', 'OP/USDT', 'ARB/USDT', 'SUI/USDT', 'SEI/USDT', 'TIA/USDT', 'MATIC/USDT', 'PEPE/USDT', 'NEAR/USDT', 'RUNE/USDT', 'PYTH/USDT', 'WIF/USDT' ]
+        self.TARGET_COINS = [ 
+            'SOL/USDT', 'XRP/USDT', 'DOGE/USDT', 'ADA/USDT', 'AVAX/USDT', 
+            'LINK/USDT', 'TON/USDT', 'INJ/USDT', 'APT/USDT', 'OP/USDT', 
+            'ARB/USDT', 'SUI/USDT', 'SEI/USDT', 'TIA/USDT', 'MATIC/USDT', 
+            'PEPE/USDT', 'NEAR/USDT', 'RUNE/USDT', 'PYTH/USDT', 'WIF/USDT' 
+        ]
         
         if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        print(f"🧪 [Backtest V69.0] Detailed Analytics Mode Active.")
+        print(f"🧪 [Backtest V71.1] 6-Frame Turbo Mode (1m Included).")
 
     # ==============================================================
     # 🛠️ Helpers
     # ==============================================================
-    def resample_data(self, df_1m, timeframe_str):
-        if df_1m.empty: return pd.DataFrame()
-        agg_dict = {'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum'}
-        rule = timeframe_str.replace('m', 'T').replace('h', 'H').replace('d', 'D')
-        try:
-            resampled = df_1m.resample(rule).agg(agg_dict).dropna()
-            resampled['timestamp'] = resampled.index.astype(np.int64) // 10**6
-            return resampled
-        except Exception: return pd.DataFrame()
-
     def df_to_list(self, df):
         if df.empty: return []
         return df[['timestamp', 'open', 'high', 'low', 'close', 'volume']].values.tolist()
 
     # ==============================================================
-    # PHASE 1: Generate Truth Data (Strict Logic Tree)
+    # PHASE 1: Generate Truth Data (Multi-Frame Vectorization)
     # ==============================================================
     async def generate_truth_data(self):
-        # (نفس منطق V68 السابق - لم يتغير لأنه صحيح)
-        print(f"\n🚜 [Phase 1] Replicating V45.0 Logic Tree ({self.BACKTEST_DAYS} Days)...")
+        print(f"\n🚜 [Phase 1] Processing Logic Tree + Titan ({self.BACKTEST_DAYS} Days)...")
         end_time_ms = int(time.time() * 1000)
         start_time_ms = end_time_ms - (self.BACKTEST_DAYS * 24 * 60 * 60 * 1000) 
         
         for sym in self.TARGET_COINS:
             safe_sym = sym.replace('/', '_')
-            scores_file = f"{CACHE_DIR}/{safe_sym}_logictree_scores.pkl"
+            scores_file = f"{CACHE_DIR}/{safe_sym}_fullstack_scores.pkl"
             
             if os.path.exists(scores_file):
                 print(f"   📂 {sym} scores ready. Skipping.")
@@ -76,6 +70,7 @@ class HeavyDutyBacktester:
 
             print(f"   ⚙️ Simulating {sym}...", end="", flush=True)
             
+            # 1. جلب بيانات الدقيقة الخام
             all_candles_1m = []
             current_since = start_time_ms
             while current_since < end_time_ms:
@@ -86,9 +81,9 @@ class HeavyDutyBacktester:
                     if last_ts <= current_since: break
                     all_candles_1m.extend(batch)
                     current_since = last_ts + 1
-                    await asyncio.sleep(0.05)
+                    await asyncio.sleep(0.01) 
                     if current_since >= end_time_ms: break
-                except: await asyncio.sleep(1)
+                except: await asyncio.sleep(0.5)
             
             all_candles_1m = [c for c in all_candles_1m if c[0] <= end_time_ms]
             if not all_candles_1m: 
@@ -100,47 +95,87 @@ class HeavyDutyBacktester:
             df_1m.set_index('datetime', inplace=True)
             df_1m = df_1m.sort_index()
             
+            # 🔥🔥🔥 التحضير المسبق (Vectorization) 🔥🔥🔥
+            agg_dict = {'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum'}
+            
+            frames = {}
+            
+            # ✅ إضافة إطار الدقيقة (1m) مباشرة
+            # لا نحتاج Resample لأنه هو الأساس، فقط نضيف عمود timestamp
+            df_1m_ready = df_1m.copy()
+            df_1m_ready['timestamp'] = df_1m_ready.index.astype(np.int64) // 10**6
+            frames['1m'] = df_1m_ready
+
+            # ✅ إضافة باقي الأطر (Resampling)
+            for tf_str, tf_code in [('5m', '5T'), ('15m', '15T'), ('1h', '1h'), ('4h', '4h'), ('1d', '1D')]:
+                resampled = df_1m.resample(tf_code).agg(agg_dict).dropna()
+                resampled['timestamp'] = resampled.index.astype(np.int64) // 10**6
+                frames[tf_str] = resampled
+
             ai_results = []
-            resample_freq = '15T'
-            time_indices = df_1m.resample(resample_freq).last().dropna().index
             
-            for t_idx in time_indices[200:]:
-                current_slice_1m = df_1m.loc[:t_idx]
-                if len(current_slice_1m) < 500: continue
-                current_price = current_slice_1m['close'].iloc[-1]
+            # نتحرك بناءً على إطار الـ 5 دقائق كقاعدة زمنية
+            valid_indices = frames['5m'].index[500:] 
+            
+            for t_idx in valid_indices:
+                current_timestamp = int(t_idx.timestamp() * 1000)
+                
+                # 🔥 قص البيانات لجميع الأطر الستة (1m included)
+                ohlcv_data = {}
                 
-                df_1h = self.resample_data(current_slice_1m.tail(6000), '1h')
-                df_15m = self.resample_data(current_slice_1m.tail(1500), '15m')
+                try:
+                    # ✅ [FIXED] إضافة 1m للحزمة (نأخذ 500 شمعة للدقة العالية)
+                    ohlcv_data['1m']  = self.df_to_list(frames['1m'].loc[:t_idx].tail(500))
+                    
+                    ohlcv_data['5m']  = self.df_to_list(frames['5m'].loc[:t_idx].tail(200))
+                    ohlcv_data['15m'] = self.df_to_list(frames['15m'].loc[:t_idx].tail(200))
+                    ohlcv_data['1h']  = self.df_to_list(frames['1h'].loc[:t_idx].tail(200))
+                    ohlcv_data['4h']  = self.df_to_list(frames['4h'].loc[:t_idx].tail(100))
+                    ohlcv_data['1d']  = self.df_to_list(frames['1d'].loc[:t_idx].tail(50))
+                except: continue 
+
+                if len(ohlcv_data['1h']) < 60: continue
                 
-                if len(df_1h) < 60 or len(df_15m) < 60: continue
+                current_price = frames['5m'].loc[t_idx]['close']
 
-                simulated_data_packet = {
+                # 1. استدعاء شجرة المنطق (Logic Tree)
+                logic_packet = {
                     'symbol': sym,
-                    'ohlcv_1h': self.df_to_list(df_1h.tail(60)),
-                    'ohlcv_15m': self.df_to_list(df_15m.tail(60)),
+                    'ohlcv_1h': ohlcv_data['1h'][-60:],   
+                    'ohlcv_15m': ohlcv_data['15m'][-60:], 
                     'change_24h': 0.0 
                 }
                 
                 try:
-                    price_24h_ago = df_1h.iloc[-24]['close'] if len(df_1h) >= 24 else df_1h.iloc[0]['close']
-                    simulated_data_packet['change_24h'] = ((current_price - price_24h_ago) / price_24h_ago) * 100
+                    if len(ohlcv_data['1h']) >= 24:
+                        p_now = ohlcv_data['1h'][-1][4]
+                        p_old = ohlcv_data['1h'][-24][4]
+                        logic_packet['change_24h'] = ((p_now - p_old) / p_old) * 100
                 except: pass
 
-                # استدعاء المنطق الصارم
-                logic_result = self.dm._apply_logic_tree(simulated_data_packet)
-                
+                logic_result = self.dm._apply_logic_tree(logic_packet)
                 signal_type = logic_result.get('type', 'NONE')
                 l1_score = logic_result.get('score', 0.0)
                 
-                # Titan Simulation Placeholder
-                titan_real = 0.5
-                
+                # 2. استدعاء Titan (Real AI)
+                real_titan = 0.5
                 if signal_type in ['BREAKOUT', 'REVERSAL']:
+                    raw_data_for_proc = {
+                        'symbol': sym,
+                        'ohlcv': ohlcv_data, # ✅ الآن تحتوي على 1m
+                        'current_price': current_price
+                    }
+                    try:
+                        proc_res = await self.proc.process_compound_signal(raw_data_for_proc)
+                        if proc_res:
+                            real_titan = proc_res.get('titan_score', 0.5)
+                    except: pass
+
                     ai_results.append({
-                        'timestamp': int(t_idx.timestamp() * 1000),
+                        'timestamp': current_timestamp,
                         'symbol': sym,
                         'close': current_price,
-                        'real_titan': titan_real, 
+                        'real_titan': real_titan, 
                         'signal_type': signal_type,
                         'l1_score': l1_score
                     })
@@ -149,10 +184,10 @@ class HeavyDutyBacktester:
                 pd.DataFrame(ai_results).to_pickle(scores_file)
                 print(f" ✅ Saved ({len(ai_results)} signals).")
             else:
-                print(" ⚠️ No strict signals found.")
+                print(" ⚠️ No signals.")
 
     # ==============================================================
-    # PHASE 2: Portfolio Digital Twin Engine (Enhanced Stats)
+    # PHASE 2: Portfolio Digital Twin Engine
     # ==============================================================
     @staticmethod
     def _worker_optimize(combinations_batch, scores_files, initial_capital, fees_pct, max_slots):
@@ -176,7 +211,7 @@ class HeavyDutyBacktester:
                 "balance": initial_capital,
                 "allocated": 0.0,
                 "positions": {}, 
-                "trades_history": [] # Store {pnl: float}
+                "trades_history": [] 
             }
             
             w_titan = config['w_titan']
@@ -237,12 +272,11 @@ class HeavyDutyBacktester:
                 
                 if wallet["balance"] < 1.0 and len(wallet["positions"]) == 0: break
 
-            # 🔥🔥🔥 إحصائيات تفصيلية (Detailed Analytics) 🔥🔥🔥
+            # Detailed Analytics
             trades = wallet["trades_history"]
             if trades:
                 net_profit = wallet["balance"] - initial_capital
                 
-                # 1. الربح والخسارة
                 pnls = [t['pnl'] for t in trades]
                 wins = [p for p in pnls if p > 0]
                 losses = [p for p in pnls if p <= 0]
@@ -252,15 +286,11 @@ class HeavyDutyBacktester:
                 total_trades = len(trades)
                 win_rate = (win_count / total_trades) * 100
                 
-                # 2. أقصى ربح وخسارة فردية
                 max_single_win = max(pnls) if pnls else 0.0
                 max_single_loss = min(pnls) if pnls else 0.0
                 
-                # 3. السلاسل المتتالية (Streaks)
-                current_win_streak = 0
-                max_win_streak = 0
-                current_loss_streak = 0
-                max_loss_streak = 0
+                current_win_streak = 0; max_win_streak = 0
+                current_loss_streak = 0; max_loss_streak = 0
                 
                 for p in pnls:
                     if p > 0:
@@ -287,13 +317,9 @@ class HeavyDutyBacktester:
                 })
             else:
                  results.append({
-                     'config': config, 
-                     'final_balance': initial_capital, 
-                     'net_profit': 0.0, 
-                     'total_trades': 0,
-                     'win_count': 0, 'loss_count': 0, 'win_rate': 0.0,
-                     'max_single_win': 0.0, 'max_single_loss': 0.0,
-                     'max_win_streak': 0, 'max_loss_streak': 0
+                     'config': config, 'final_balance': initial_capital, 'net_profit': 0.0, 
+                     'total_trades': 0, 'win_count': 0, 'loss_count': 0, 'win_rate': 0.0,
+                     'max_single_win': 0.0, 'max_single_loss': 0.0, 'max_win_streak': 0, 'max_loss_streak': 0
                  })
 
         return results
@@ -301,12 +327,12 @@ class HeavyDutyBacktester:
     async def run_optimization(self):
         await self.generate_truth_data()
         
-        score_files = [os.path.join(CACHE_DIR, f) for f in os.listdir(CACHE_DIR) if f.endswith(f'_logictree_scores.pkl')]
+        score_files = [os.path.join(CACHE_DIR, f) for f in os.listdir(CACHE_DIR) if f.endswith(f'_fullstack_scores.pkl')]
         if not score_files:
-            print("❌ No Strict Logic signals found.")
+            print("❌ No Full-Stack signals found.")
             return None
 
-        print(f"\n🧩 [Phase 2] Running Strict Logic Simulation...")
+        print(f"\n🧩 [Phase 2] Running Full-Stack Simulation...")
         print(f"   💰 Start Capital: ${self.INITIAL_CAPITAL}")
         
         w_titan_range = np.linspace(0.4, 0.9, num=self.GRID_DENSITY)
@@ -338,9 +364,8 @@ class HeavyDutyBacktester:
             
         best = sorted(final_results, key=lambda x: x['final_balance'], reverse=True)[0]
         
-        # 🔥🔥🔥 التقرير التفصيلي الكامل 🔥🔥🔥
         print("\n" + "="*60)
-        print(f"🏆 CHAMPION STRICT REPORT ({self.BACKTEST_DAYS} Days):")
+        print(f"🏆 CHAMPION FULL-STACK REPORT ({self.BACKTEST_DAYS} Days):")
         print(f"   💰 Final Balance:   ${best['final_balance']:,.2f}")
         print(f"   🚀 Net PnL:         ${best['net_profit']:,.2f}")
         print("-" * 60)
@@ -360,7 +385,7 @@ class HeavyDutyBacktester:
         return best['config']
 
 async def run_strategic_optimization_task():
-    print("\n🧪 [STRATEGIC BACKTEST] Starting Logic Tree Optimization...")
+    print("\n🧪 [STRATEGIC BACKTEST] Starting Full-Stack Optimization...")
     r2 = R2Service()
     dm = DataManager(None, None, r2)
     proc = MLProcessor(dm)