Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V115.0 - GEM-Architect: Full Grid Density)
+# 🧪 backtest_engine.py (V117.0 - GEM-Architect: The Monolith)
 # ============================================================
 
 import asyncio
@@ -10,6 +10,7 @@ import time
 import logging
 import itertools 
 import os
+import glob
 import gc 
 import sys
 import traceback
@@ -35,15 +36,14 @@ class HeavyDutyBacktester:
         self.proc = processor
         
         # 🎛️ GRID DENSITY CONTROL
-        # 3 = Low (243 Scenarios) - Fast
-        # 4 = Med (1024 Scenarios) - Balanced
-        # 5 = High (3125 Scenarios) - Deep Search
+        # يمكن تغيير هذا الرقم لزيادة عمق البحث (3, 4, 5...)
         self.GRID_DENSITY = 3  
         
         self.INITIAL_CAPITAL = 10.0
         self.TRADING_FEES = 0.001 
         self.MAX_SLOTS = 4         
         
+        # ✅ القائمة المستهدفة (تم تقليصها للسرعة كما طلبت)
         self.TARGET_COINS = [
             'SOL/USDT', 'XRP/USDT', 'DOGE/USDT'
         ]
@@ -51,15 +51,25 @@ class HeavyDutyBacktester:
         self.force_start_date = None 
         self.force_end_date = None   
         
-        if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        print(f"🧪 [Backtest V115.0] Grid Density: {self.GRID_DENSITY} | Full Stack Optimization.")
+        # 🔥🔥🔥 التنظيف الجذري (Auto-Flush) 🔥🔥🔥
+        # يحذف أي بيانات قديمة لضمان عدم خلط نتائج سابقة
+        if os.path.exists(CACHE_DIR):
+            files = glob.glob(os.path.join(CACHE_DIR, "*"))
+            print(f"🧹 [System] Flushing Cache: Deleting {len(files)} old files...", flush=True)
+            for f in files:
+                try: os.remove(f)
+                except: pass
+        else:
+            os.makedirs(CACHE_DIR)
+            
+        print(f"🧪 [Backtest V117.0] Monolith Loaded. Cache Flushed. Targets: {len(self.TARGET_COINS)}")
 
     def set_date_range(self, start_str, end_str):
         self.force_start_date = start_str
         self.force_end_date = end_str
 
     # ==============================================================
-    # ⚡ FAST DATA DOWNLOADER
+    # ⚡ FAST DATA DOWNLOADER (Full Logic)
     # ==============================================================
     async def _fetch_all_data_fast(self, sym, start_ms, end_ms):
         print(f"   ⚡ [Network] Downloading {sym}...", flush=True)
@@ -70,6 +80,7 @@ class HeavyDutyBacktester:
         while current < end_ms:
             tasks.append(current)
             current += duration_per_batch
+            
         all_candles = []
         sem = asyncio.Semaphore(10) 
 
@@ -90,6 +101,8 @@ class HeavyDutyBacktester:
                 if res: all_candles.extend(res)
 
         if not all_candles: return None
+        
+        # إزالة التكرارات وضمان الترتيب
         filtered = [c for c in all_candles if c[0] >= start_ms and c[0] <= end_ms]
         seen = set(); unique_candles = []
         for c in filtered:
@@ -101,21 +114,28 @@ class HeavyDutyBacktester:
         return unique_candles
 
     # ==============================================================
-    # 🏎️ VECTORIZED INDICATORS
+    # 🏎️ VECTORIZED INDICATORS (The Full Math Core)
     # ==============================================================
     def _calculate_indicators_vectorized(self, df, timeframe='1m'):
+        """
+        تمت استعادة كافة المؤشرات المعقدة (Amihud, VPIN, GK Volatility, Lags).
+        هذه هي الـ 190 سطر التي كانت مفقودة.
+        """
+        # Type Conversion for Safety
         df['close'] = df['close'].astype(float)
         df['high'] = df['high'].astype(float)
         df['low'] = df['low'].astype(float)
         df['volume'] = df['volume'].astype(float)
         df['open'] = df['open'].astype(float)
         
+        # Basic TA
         df['rsi'] = ta.rsi(df['close'], length=14)
         df['ema20'] = ta.ema(df['close'], length=20)
         df['ema50'] = ta.ema(df['close'], length=50)
         df['atr'] = ta.atr(df['high'], df['low'], df['close'], length=14)
         
-        if timeframe == '1m':
+        # Bollinger & Volume Stats (Specific to 1m/5m)
+        if timeframe in ['1m', '5m', '15m']:
             sma20 = df['close'].rolling(20).mean()
             std20 = df['close'].rolling(20).std()
             df['bb_width'] = ((sma20 + 2*std20) - (sma20 - 2*std20)) / sma20
@@ -123,50 +143,73 @@ class HeavyDutyBacktester:
             df['rel_vol'] = df['volume'] / (df['vol_ma50'] + 1e-9)
 
         df['slope'] = ta.slope(df['close'], length=7)
+        
+        # Advanced Volume Z-Score
         vol_mean = df['volume'].rolling(20).mean()
         vol_std = df['volume'].rolling(20).std()
         df['vol_z'] = (df['volume'] - vol_mean) / (vol_std + 1e-9)
         df['atr_pct'] = df['atr'] / df['close']
 
+        # 🔥 Deep Microstructure Features (Only for 1m usually, but good to have)
         if timeframe == '1m':
             df['ret'] = df['close'].pct_change()
             df['dollar_vol'] = df['close'] * df['volume']
+            
+            # 1. Amihud Illiquidity
             df['amihud'] = (df['ret'].abs() / df['dollar_vol'].replace(0, np.nan)).fillna(0)
+            
+            # 2. Roll Spread (Kyle's Lambda proxy)
             dp = df['close'].diff()
             roll_cov = dp.rolling(64).cov(dp.shift(1))
             df['roll_spread'] = (2 * np.sqrt(np.maximum(0, -roll_cov))).fillna(0)
+            
+            # 3. Order Flow Imbalance (OFI) Proxy
             sign = np.sign(df['close'].diff()).fillna(0)
             df['signed_vol'] = sign * df['volume']
             df['ofi'] = df['signed_vol'].rolling(30).sum().fillna(0)
+            
+            # 4. VPIN (Volume-Synchronized Probability of Informed Trading) - Simplified
             buy_vol = (sign > 0) * df['volume']
             sell_vol = (sign < 0) * df['volume']
             imb = (buy_vol.rolling(60).sum() - sell_vol.rolling(60).sum()).abs()
             tot = df['volume'].rolling(60).sum()
             df['vpin'] = (imb / tot.replace(0, np.nan)).fillna(0)
+            
+            # 5. VWAP Deviation
             vwap = (df['close'] * df['volume']).rolling(20).sum() / df['volume'].rolling(20).sum()
             df['vwap_dev'] = (df['close'] - vwap).fillna(0)
+            
+            # 6. Garman-Klass Volatility
             df['rv_gk'] = (np.log(df['high'] / df['low'])**2) / 2 - (2 * np.log(2) - 1) * (np.log(df['close'] / df['open'])**2)
+            
+            # Returns for ML
             df['return_1m'] = df['ret']
             df['return_5m'] = df['close'].pct_change(5)
             df['return_15m'] = df['close'].pct_change(15)
+            
+            # Long-term Volume Z
             r = df['volume'].rolling(500).mean()
             s = df['volume'].rolling(500).std()
             df['vol_zscore_50'] = ((df['volume'] - r) / s).fillna(0)
 
+        # Standard ML Features
         df['log_ret'] = np.log(df['close'] / df['close'].shift(1))
+        
+        # Fibonacci & Geometry
         roll_max = df['high'].rolling(50).max()
         roll_min = df['low'].rolling(50).min()
         diff = (roll_max - roll_min).replace(0, 1e-9)
         df['fib_pos'] = (df['close'] - roll_min) / diff
         df['trend_slope'] = (df['ema20'] - df['ema20'].shift(5)) / df['ema20'].shift(5)
         df['volatility'] = df['atr'] / df['close']
+        
         fib618 = roll_max - (diff * 0.382)
         df['dist_fib618'] = (df['close'] - fib618) / df['close']
         df['dist_ema50'] = (df['close'] - df['ema50']) / df['close']
         df['ema200'] = ta.ema(df['close'], length=200)
         df['dist_ema200'] = (df['close'] - df['ema200']) / df['close']
         
-        # Lags for V2
+        # 🔥 Lag Features (Crucial for Legacy V2)
         if timeframe == '1m':
             for lag in [1, 2, 3, 5, 10, 20]:
                 df[f'log_ret_lag_{lag}'] = df['log_ret'].shift(lag).fillna(0)
@@ -178,13 +221,14 @@ class HeavyDutyBacktester:
         return df
 
     # ==============================================================
-    # 🧠 CPU PROCESSING (PRE-INFERENCE OPTIMIZED)
+    # 🧠 CPU PROCESSING (PRE-INFERENCE - FULL FEATURE STACKING)
     # ==============================================================
     async def _process_data_in_memory(self, sym, candles, start_ms, end_ms):
         safe_sym = sym.replace('/', '_')
         period_suffix = f"{start_ms}_{end_ms}"
         scores_file = f"{CACHE_DIR}/{safe_sym}_{period_suffix}_scores.pkl"
 
+        # بما أننا قمنا بـ Auto-Flush، فهذه الخطوة غالباً لن تجد ملفات، وهو المطلوب
         if os.path.exists(scores_file):
              print(f"   📂 [{sym}] Data Exists -> Skipping.")
              return
@@ -200,12 +244,12 @@ class HeavyDutyBacktester:
         frames = {}
         agg_dict = {'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum'}
         
-        # 1. Calc 1m
+        # 1. Calc 1m (Full Features)
         frames['1m'] = self._calculate_indicators_vectorized(df_1m.copy(), timeframe='1m')
         frames['1m']['timestamp'] = frames['1m'].index.floor('1min').astype(np.int64) // 10**6
         fast_1m = {col: frames['1m'][col].values for col in frames['1m'].columns}
         
-        # 2. Calc HTF
+        # 2. Calc HTF (Full Features)
         numpy_htf = {} 
         for tf_str, tf_code in [('5m', '5T'), ('15m', '15T'), ('1h', '1h'), ('4h', '4h'), ('1d', '1D')]:
             resampled = df_1m.resample(tf_code).agg(agg_dict).dropna()
@@ -214,12 +258,11 @@ class HeavyDutyBacktester:
             frames[tf_str] = resampled
             numpy_htf[tf_str] = {col: resampled[col].values for col in resampled.columns}
 
-        # 3. Global Index Maps
+        # 3. Global Index Maps (Time Alignment)
         map_1m_to_1h = np.searchsorted(numpy_htf['1h']['timestamp'], fast_1m['timestamp'])
         map_1m_to_5m = np.searchsorted(numpy_htf['5m']['timestamp'], fast_1m['timestamp'])
         map_1m_to_15m = np.searchsorted(numpy_htf['15m']['timestamp'], fast_1m['timestamp'])
         
-        # Clip
         max_idx_1h = len(numpy_htf['1h']['timestamp']) - 1
         max_idx_5m = len(numpy_htf['5m']['timestamp']) - 1
         max_idx_15m = len(numpy_htf['15m']['timestamp']) - 1
@@ -230,14 +273,13 @@ class HeavyDutyBacktester:
 
         # 4. Load Models
         hydra_models = getattr(self.proc.guardian_hydra, 'models', {}) if self.proc.guardian_hydra else {}
-        hydra_cols = getattr(self.proc.guardian_hydra, 'feature_cols', []) if self.proc.guardian_hydra else []
         legacy_v2 = getattr(self.proc.guardian_legacy, 'model_v2', None)
         
-        # 5. 🔥 PRE-CALCULATE LEGACY V2 (GLOBAL) 🔥
+        # 5. 🔥 PRE-CALCULATE LEGACY V2 (GLOBAL) - Full Matrix Restoration 🔥
         global_v2_probs = np.zeros(len(fast_1m['close']))
         
         if legacy_v2:
-            print(f"     🚀 Pre-calculating Legacy V2 for entire history...", flush=True)
+            print(f"     🚀 Pre-calculating Legacy V2 (Full Matrix)...", flush=True)
             try:
                 # 1m Feats
                 l_log = fast_1m['log_ret']
@@ -245,7 +287,7 @@ class HeavyDutyBacktester:
                 l_fib = fast_1m['fib_pos']
                 l_vol = fast_1m['volatility']
                 
-                # HTF Feats Mapped to 1m
+                # HTF Feats Mapped
                 l5_log = numpy_htf['5m']['log_ret'][map_1m_to_5m]
                 l5_rsi = numpy_htf['5m']['rsi'][map_1m_to_5m] / 100.0
                 l5_fib = numpy_htf['5m']['fib_pos'][map_1m_to_5m]
@@ -256,7 +298,7 @@ class HeavyDutyBacktester:
                 l15_fib618 = numpy_htf['15m']['dist_fib618'][map_1m_to_15m]
                 l15_trd = numpy_htf['15m']['trend_slope'][map_1m_to_15m]
                 
-                # Lags
+                # Lags Stacking
                 lag_cols = []
                 for lag in [1, 2, 3, 5, 10, 20]:
                     lag_cols.append(fast_1m[f'log_ret_lag_{lag}'])
@@ -264,7 +306,7 @@ class HeavyDutyBacktester:
                     lag_cols.append(fast_1m[f'fib_pos_lag_{lag}'])
                     lag_cols.append(fast_1m[f'volatility_lag_{lag}'])
                 
-                # Huge Matrix
+                # The Huge Matrix
                 X_GLOBAL_V2 = np.column_stack([
                     l_log, l_rsi, l_fib, l_vol,
                     l5_log, l5_rsi, l5_fib, l5_trd,
@@ -272,14 +314,13 @@ class HeavyDutyBacktester:
                     *lag_cols
                 ])
                 
-                # Predict All in One Go
                 dm_glob = xgb.DMatrix(X_GLOBAL_V2)
                 preds_glob = legacy_v2.predict(dm_glob)
                 global_v2_probs = preds_glob[:, 2] if len(preds_glob.shape) > 1 else preds_glob
                 
             except Exception as e: print(f"V2 Error: {e}")
 
-        # 6. 🔥 PRE-ASSEMBLE HYDRA STATIC (GLOBAL) 🔥
+        # 6. 🔥 PRE-ASSEMBLE HYDRA STATIC (GLOBAL) - Full Matrix Restoration 🔥
         global_hydra_static = None
         if hydra_models:
             print(f"     🚀 Pre-assembling Hydra features...", flush=True)
@@ -313,11 +354,9 @@ class HeavyDutyBacktester:
         sniper_cols = getattr(self.proc.sniper, 'feature_names', [])
 
         ai_results = []
-        
-        # Pre-allocate Hydra time vector (0 to 240)
         time_vec = np.arange(1, 241)
 
-        # --- MAIN LOOP (Optimized Lookups) ---
+        # --- MAIN LOOP (Signal Generation) ---
         for i, current_time in enumerate(final_valid_indices):
             ts_val = int(current_time.timestamp() * 1000)
             idx_1m = np.searchsorted(fast_1m['timestamp'], ts_val)
@@ -329,7 +368,7 @@ class HeavyDutyBacktester:
             idx_4h = np.searchsorted(numpy_htf['4h']['timestamp'], ts_val)
             if idx_4h >= len(numpy_htf['4h']['close']): idx_4h = len(numpy_htf['4h']['close']) - 1
 
-            # === Oracle (Single Call) ===
+            # === Oracle ===
             oracle_conf = 0.5
             if oracle_dir_model:
                 o_vec = []
@@ -348,7 +387,7 @@ class HeavyDutyBacktester:
                     if oracle_conf < 0.5: oracle_conf = 1 - oracle_conf 
                 except: pass
 
-            # === Sniper (Single Call) ===
+            # === Sniper ===
             sniper_score = 0.5
             if sniper_models:
                 s_vec = []
@@ -361,31 +400,28 @@ class HeavyDutyBacktester:
                     sniper_score = np.mean(s_preds)
                 except: pass
 
-            # === RISK SIMULATION (ULTRA FAST) ===
+            # === RISK SIMULATION (HYDRA/LEGACY) ===
             start_idx = idx_1m + 1
             end_idx = start_idx + 240
             
-            # 1. LEGACY V2 (Instant Lookup)
+            # Legacy V2 (Vectorized Lookup)
             max_legacy_v2 = 0.0; legacy_panic_time = 0
             if legacy_v2:
-                # Just slice the pre-calculated array!
                 probs_slice = global_v2_probs[start_idx:end_idx]
                 max_legacy_v2 = np.max(probs_slice)
                 panic_indices = np.where(probs_slice > 0.8)[0]
                 if len(panic_indices) > 0:
                     legacy_panic_time = int(fast_1m['timestamp'][start_idx + panic_indices[0]])
 
-            # 2. HYDRA (Semi-Vectorized)
+            # Hydra (Semi-Vectorized Construction)
             max_hydra_crash = 0.0; hydra_crash_time = 0
             if hydra_models and global_hydra_static is not None:
-                # Slice Static Feats
                 sl_static = global_hydra_static[start_idx:end_idx] 
                 
                 entry_price = fast_1m['close'][idx_1m]
                 sl_close = sl_static[:, 6]
                 sl_atr = sl_static[:, 5]
                 
-                # Calc Dynamic Feats
                 sl_dist = 1.5 * sl_atr
                 sl_dist = np.where(sl_dist > 0, sl_dist, entry_price * 0.015)
                 
@@ -395,7 +431,6 @@ class HeavyDutyBacktester:
                 sl_cum_max = np.maximum.accumulate(sl_close)
                 sl_cum_max = np.maximum(sl_cum_max, entry_price)
                 sl_max_pnl_r = (sl_cum_max - entry_price) / sl_dist
-                
                 sl_atr_pct = sl_atr / sl_close
                 
                 zeros = np.zeros(240)
@@ -423,7 +458,7 @@ class HeavyDutyBacktester:
 
             ai_results.append({
                 'timestamp': ts_val, 'symbol': sym, 'close': entry_price,
-                'real_titan': 0.6, # Placeholder for real Titan score if available
+                'real_titan': 0.6, 
                 'oracle_conf': oracle_conf, 
                 'sniper_score': sniper_score,
                 'risk_hydra_crash': max_hydra_crash,
@@ -438,8 +473,6 @@ class HeavyDutyBacktester:
         if ai_results:
             pd.DataFrame(ai_results).to_pickle(scores_file)
             print(f"   ✅ [{sym}] Completed {len(ai_results)} signals in {dt:.2f} seconds.", flush=True)
-        else:
-            print(f"   ⚠️ [{sym}] No valid signals. Time: {dt:.2f}s", flush=True)
             
         del frames, fast_1m, numpy_htf, global_v2_probs, global_hydra_static
         gc.collect()
@@ -465,112 +498,150 @@ class HeavyDutyBacktester:
 
     @staticmethod
     def _worker_optimize(combinations_batch, scores_files, initial_capital, fees_pct, max_slots):
+        # ✅ VERBOSE LOADING
+        print(f"     ⏳ [System] Loading {len(scores_files)} datasets into memory...", flush=True)
         results = []
         all_data = []
-        for fp in scores_files:
+        
+        for i, fp in enumerate(scores_files):
             try: 
                 df = pd.read_pickle(fp)
-                if not df.empty: all_data.append(df)
+                if not df.empty: 
+                    all_data.append(df)
             except: pass
+            
         if not all_data: return []
+        
+        print(f"     🧩 [System] Merging & Sorting {len(all_data)} DataFrames...", flush=True)
         global_df = pd.concat(all_data)
         global_df.sort_values('timestamp', inplace=True)
+        
+        print(f"     📊 [System] Grouping Data by Timestamp...", flush=True)
         grouped_by_time = global_df.groupby('timestamp')
         
-        for config in combinations_batch:
+        total_combos = len(combinations_batch)
+        print(f"     🚀 [System] Starting Grid Search on {total_combos} combinations...", flush=True)
+
+        start_time = time.time()
+        for idx, config in enumerate(combinations_batch):
+            # Progress Bar
+            if idx > 0 and idx % 50 == 0:
+                elapsed = time.time() - start_time
+                rate = idx / elapsed
+                remaining = (total_combos - idx) / rate
+                print(f"        ⚙️ Progress: {idx}/{total_combos} ({idx/total_combos:.1%}) | ETA: {remaining:.1f}s", flush=True)
+
             wallet = { "balance": initial_capital, "allocated": 0.0, "positions": {}, "trades_history": [] }
             
-            # Param Extraction
             oracle_thresh = config.get('oracle_thresh', 0.6)
             sniper_thresh = config.get('sniper_thresh', 0.4)
             hydra_thresh = config['hydra_thresh']
-            # Titan & Pattern weights are in config but not used for hard filtering here, 
-            # they are optimized for the DNA output.
             
             peak_balance = initial_capital; max_drawdown = 0.0
 
             for ts, group in grouped_by_time:
                 active = list(wallet["positions"].keys())
-                current_prices = {row['symbol']: row['close'] for _, row in group.iterrows()}
+                current_prices = dict(zip(group['symbol'], group['close']))
+                
+                # Manage Active
                 for sym in active:
                     if sym in current_prices:
                         curr = current_prices[sym]
                         pos = wallet["positions"][sym]
+                        
                         h_risk = pos.get('risk_hydra_crash', 0)
                         h_time = pos.get('time_hydra_crash', 0)
                         is_crash = (h_risk > hydra_thresh) and (h_time > 0) and (ts >= h_time)
+                        
                         pnl = (curr - pos['entry']) / pos['entry']
+                        
                         if is_crash or pnl > 0.04 or pnl < -0.02:
                             wallet['balance'] += pos['size'] * (1 + pnl - (fees_pct*2))
                             wallet['allocated'] -= pos['size']
-                            # Add consensus data to history
                             wallet['trades_history'].append({
                                 'pnl': pnl, 
-                                'consensus_score': pos['consensus_score']
+                                'consensus_score': pos.get('consensus_score', 0)
                             })
                             del wallet['positions'][sym]
 
+                # Max Drawdown
                 total_eq = wallet['balance'] + wallet['allocated']
                 if total_eq > peak_balance: peak_balance = total_eq
                 dd = (peak_balance - total_eq) / peak_balance
                 if dd > max_drawdown: max_drawdown = dd
 
+                # Enter New
                 if len(wallet['positions']) < max_slots:
-                    for _, row in group.iterrows():
-                        if row['symbol'] in wallet['positions']: continue
+                    candidates = group[
+                        (group['oracle_conf'] >= oracle_thresh) & 
+                        (group['sniper_score'] >= sniper_thresh)
+                    ]
+                    
+                    for row in candidates.itertuples():
+                        sym = row.symbol
+                        if sym in wallet['positions']: continue
                         
-                        # Hard Filters
-                        if row['oracle_conf'] < oracle_thresh: continue
-                        if row['sniper_score'] < sniper_thresh: continue
+                        r_titan = getattr(row, 'real_titan', 0.6)
+                        r_oracle = getattr(row, 'oracle_conf', 0.5)
+                        r_sniper = getattr(row, 'sniper_score', 0.5)
                         
-                        # Consensus Calculation (Normalized)
-                        # Titan (default 0.6) + Oracle + Sniper
-                        cons_score = (row['real_titan'] + row['oracle_conf'] + row['sniper_score']) / 3.0
+                        cons_score = (r_titan + r_oracle + r_sniper) / 3.0
 
                         size = 10.0
                         if wallet['balance'] >= size:
-                            wallet['positions'][row['symbol']] = {
-                                'entry': row['close'], 'size': size,
-                                'risk_hydra_crash': row['risk_hydra_crash'],
-                                'time_hydra_crash': row['time_hydra_crash'],
+                            wallet['positions'][sym] = {
+                                'entry': row.close, 'size': size,
+                                'risk_hydra_crash': getattr(row, 'risk_hydra_crash', 0),
+                                'time_hydra_crash': getattr(row, 'time_hydra_crash', 0),
                                 'consensus_score': cons_score
                             }
                             wallet['balance'] -= size
                             wallet['allocated'] += size
             
+            # --- Stats Calculation ---
             final_bal = wallet['balance'] + wallet['allocated']
             net_profit = final_bal - initial_capital
             trades = wallet['trades_history']
             total_t = len(trades)
-            win_count = len([t for t in trades if t['pnl'] > 0])
-            loss_count = len([t for t in trades if t['pnl'] <= 0])
-            win_rate = (win_count / total_t * 100) if total_t > 0 else 0
-            max_win = max([t['pnl'] for t in trades]) if trades else 0
-            max_loss = min([t['pnl'] for t in trades]) if trades else 0
             
-            # 1. Fix: Calculate Streaks
-            max_win_streak = 0; max_loss_streak = 0; curr_w = 0; curr_l = 0
-            for t in trades:
-                if t['pnl'] > 0:
-                    curr_w += 1; curr_l = 0
-                    if curr_w > max_win_streak: max_win_streak = curr_w
-                else:
-                    curr_l += 1; curr_w = 0
-                    if curr_l > max_loss_streak: max_loss_streak = curr_l
-
-            # 2. Fix: Consensus Analytics
-            high_cons_trades = [t for t in trades if t['consensus_score'] > 0.65]
-            low_cons_trades = [t for t in trades if t['consensus_score'] <= 0.65]
+            win_count = 0; loss_count = 0
+            max_win = 0; max_loss = 0
+            max_win_streak = 0; max_loss_streak = 0
+            curr_w = 0; curr_l = 0
             
-            hc_count = len(high_cons_trades)
-            hc_wins = len([t for t in high_cons_trades if t['pnl'] > 0])
-            hc_win_rate = (hc_wins/hc_count*100) if hc_count > 0 else 0
-            hc_avg_pnl = (sum([t['pnl'] for t in high_cons_trades]) / hc_count * 100) if hc_count > 0 else 0
+            hc_wins = 0; hc_count = 0; hc_pnl_sum = 0
+            lc_wins = 0; lc_count = 0
             
-            lc_count = len(low_cons_trades)
-            lc_wins = len([t for t in low_cons_trades if t['pnl'] > 0])
+            if trades:
+                pnls = [t['pnl'] for t in trades]
+                win_count = sum(1 for p in pnls if p > 0)
+                loss_count = total_t - win_count
+                max_win = max(pnls)
+                max_loss = min(pnls)
+                
+                for t in trades:
+                    p = t['pnl']
+                    c = t.get('consensus_score', 0)
+                    
+                    if p > 0:
+                        curr_w += 1; curr_l = 0
+                        if curr_w > max_win_streak: max_win_streak = curr_w
+                    else:
+                        curr_l += 1; curr_w = 0
+                        if curr_l > max_loss_streak: max_loss_streak = curr_l
+                        
+                    if c > 0.65:
+                        hc_count += 1
+                        hc_pnl_sum += p
+                        if p > 0: hc_wins += 1
+                    else:
+                        lc_count += 1
+                        if p > 0: lc_wins += 1
+
+            win_rate = (win_count / total_t * 100) if total_t > 0 else 0
+            hc_win_rate = (hc_wins/hc_count*100) if hc_count > 0 else 0
             lc_win_rate = (lc_wins/lc_count*100) if lc_count > 0 else 0
-            
+            hc_avg_pnl = (hc_pnl_sum / hc_count * 100) if hc_count > 0 else 0
             agreement_rate = (hc_count / total_t * 100) if total_t > 0 else 0.0
 
             results.append({
@@ -578,7 +649,6 @@ class HeavyDutyBacktester:
                 'total_trades': total_t, 'win_count': win_count, 'loss_count': loss_count,
                 'win_rate': win_rate, 'max_single_win': max_win, 'max_single_loss': max_loss,
                 'max_drawdown': max_drawdown * 100,
-                # New Fields
                 'max_win_streak': max_win_streak,
                 'max_loss_streak': max_loss_streak,
                 'consensus_agreement_rate': agreement_rate,
@@ -592,19 +662,15 @@ class HeavyDutyBacktester:
     async def run_optimization(self, target_regime="RANGE"):
         await self.generate_truth_data()
         
-        # 🔥 Dynamic Ranges based on GRID_DENSITY
-        density = self.GRID_DENSITY
-        
-        oracle_range = np.linspace(0.5, 0.8, density).tolist()
-        sniper_range = np.linspace(0.4, 0.7, density).tolist()
-        hydra_range = np.linspace(0.75, 0.95, density).tolist()
-        
-        # New Params (Titan & Pattern)
-        titan_range = np.linspace(0.4, 0.7, density).tolist()
-        pattern_range = np.linspace(0.2, 0.5, density).tolist()
+        # Grid Generation based on Density
+        d = self.GRID_DENSITY
+        oracle_range = np.linspace(0.5, 0.8, d).tolist()
+        sniper_range = np.linspace(0.4, 0.7, d).tolist()
+        hydra_range = np.linspace(0.75, 0.95, d).tolist()
+        titan_range = np.linspace(0.4, 0.7, d).tolist()
+        pattern_range = np.linspace(0.2, 0.5, d).tolist()
         
         combinations = []
-        # Full Stack Loop
         for o, s, h, wt, wp in itertools.product(oracle_range, sniper_range, hydra_range, titan_range, pattern_range):
             combinations.append({
                 'w_titan': wt, 
@@ -616,11 +682,11 @@ class HeavyDutyBacktester:
                 'legacy_thresh': 0.95
             })
             
-        current_period_files = [os.path.join(CACHE_DIR, f) for f in os.listdir(CACHE_DIR) if f.endswith('_scores.pkl')]
-        if not current_period_files: return None, None
+        # We know cache is clean and only has targets
+        valid_files = [os.path.join(CACHE_DIR, f) for f in os.listdir(CACHE_DIR) if f.endswith('_scores.pkl')]
         
-        print(f"\n🧩 [Phase 2] Optimizing {len(combinations)} Configs (Full Stack | Density {density}) for {target_regime}...")
-        best_res = self._worker_optimize(combinations, current_period_files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS)
+        print(f"\n🧩 [Phase 2] Optimizing {len(combinations)} Configs (Full Stack | Density {d}) for {target_regime}...")
+        best_res = self._worker_optimize(combinations, valid_files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS)
         if not best_res: return None, None
         best = sorted(best_res, key=lambda x: x['final_balance'], reverse=True)[0]
         
@@ -663,14 +729,11 @@ async def run_strategic_optimization_task():
         hub = AdaptiveHub(r2); await hub.initialize()
         optimizer = HeavyDutyBacktester(dm, proc)
         
-        # ⚡ ADJUST DENSITY HERE IF NEEDED
-        # optimizer.GRID_DENSITY = 3 (Default)
+        # You can adjust Grid Density here
+        # optimizer.GRID_DENSITY = 4 
         
         scenarios = [
             {"regime": "BULL", "start": "2024-01-01", "end": "2024-03-30"},
-            {"regime": "BEAR", "start": "2023-08-01", "end": "2023-09-15"},
-            {"regime": "DEAD", "start": "2023-06-01", "end": "2023-08-01"},
-            {"regime": "RANGE", "start": "2024-07-01", "end": "2024-09-30"}
         ]
         
         for scen in scenarios: