Update backtest_engine.py

backtest_engine.py

@@ -1,5 +1,5 @@
 # ============================================================
-# 🧪 backtest_engine.py (V160.0 - GEM-Architect: Gov + L1 Update)
+# 🧪 backtest_engine.py (V165.0 - GEM-Architect: Coin States & Kill-Switch)
 # ============================================================
 
 import asyncio
@@ -59,19 +59,19 @@ class HeavyDutyBacktester:
         self.dm = data_manager
         self.proc = processor
         
-        # 🎛️ الكثافة (Density): عدد الخطوات في النطاق
+        # 🎛️ الكثافة (Density)
         self.GRID_DENSITY = 3 
         
         self.INITIAL_CAPITAL = 10.0
         self.TRADING_FEES = 0.001 
         self.MAX_SLOTS = 4
         
-        # 🎛️ CONTROL PANEL - DYNAMIC RANGES
+        # 🎛️ CONTROL PANEL
         self.GRID_RANGES = {
             'TITAN':  np.linspace(0.20, 0.50, self.GRID_DENSITY),
             'ORACLE': np.linspace(0.50, 0.80, self.GRID_DENSITY),
             'SNIPER': np.linspace(0.30, 0.70, self.GRID_DENSITY),
-            'GOV_SCORE': np.linspace(50.0, 80.0, self.GRID_DENSITY), # ✅ Added Governance Threshold
+            'GOV_SCORE': np.linspace(50.0, 75.0, self.GRID_DENSITY),
         }
         
         self.TARGET_COINS = [
@@ -86,13 +86,13 @@ class HeavyDutyBacktester:
         ]
         
         # ✅ DATE SETTINGS
-        self.USE_FIXED_DATES = False  # Set to True to use force_start_date
-        self.LOOKBACK_DAYS = 30       # Default lookback
+        self.USE_FIXED_DATES = False  
+        self.LOOKBACK_DAYS = 30       
         self.force_start_date = "2024-01-01" 
         self.force_end_date = "2024-02-01"   
         
         if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR)
-        print(f"🧪 [Backtest V160.0] Hyper-Speed Jump Engine (Gov + L1 Update).")
+        print(f"🧪 [Backtest V165.0] Coin States & Kill-Switch System.")
 
     def set_date_range(self, start_str, end_str):
         self.force_start_date = start_str
@@ -131,14 +131,14 @@ class HeavyDutyBacktester:
         return df.values.tolist()
 
     # ----------------------------------------------------------------------
-    # 🏎️ VECTORIZED INDICATORS (Enhanced for L1 & Gov)
+    # 🏎️ VECTORIZED INDICATORS (Enhanced for Coin Types)
     # ----------------------------------------------------------------------
     def _calculate_indicators_vectorized(self, df, timeframe='1m'):
         if df.empty: return df
         cols = ['close', 'high', 'low', 'volume', 'open']
         for c in cols: df[c] = df[c].astype(np.float64)
         
-        # Basic EMAs (Required for L1 & Gov)
+        # EMAs
         df['ema9'] = df['close'].ewm(span=9, adjust=False).mean()
         df['ema20'] = df['close'].ewm(span=20, adjust=False).mean()
         df['ema21'] = df['close'].ewm(span=21, adjust=False).mean()
@@ -159,33 +159,36 @@ class HeavyDutyBacktester:
             macd = ta.macd(df['close'])
             if macd is not None:
                 df['MACD'] = macd.iloc[:, 0].fillna(0)
-                df['MACD_s'] = macd.iloc[:, 2].fillna(0) # Signal line
+                df['MACD_s'] = macd.iloc[:, 2].fillna(0)
                 df['MACD_h'] = macd.iloc[:, 1].fillna(0)
             else: df['MACD'] = 0; df['MACD_h'] = 0; df['MACD_s'] = 0
             
+            # ✅ Essential for Market Kill-Switch & Trash Filter
             df['ADX'] = ta.adx(df['high'], df['low'], df['close'], length=14).iloc[:, 0].fillna(0)
+            df['CHOP'] = ta.chop(df['high'], df['low'], df['close'], length=14).fillna(50)
+            
             df['CCI'] = ta.cci(df['high'], df['low'], df['close'], length=20).fillna(0)
             df['MFI'] = ta.mfi(df['high'], df['low'], df['close'], df['volume'], length=14).fillna(50)
-            df['slope'] = ta.slope(df['close'], length=7).fillna(0)
             vwap = ta.vwap(df['high'], df['low'], df['close'], df['volume'])
             df['vwap'] = vwap.fillna(df['close']) if vwap is not None else df['close']
 
         c = df['close'].values
-        # Distances for L1 Logic
-        df['dist_ema50'] = (df['ema50'] - c) / df['ema50'] # Positive if price below ema50
+        # Distances
+        df['dist_ema50'] = (df['ema50'] - c) / df['ema50']
         df['dist_upper'] = (df['upper_bb'] - c) / c
         
-        # Features for models
+        # Features
         df['EMA_9_dist'] = (c / df['ema9'].values) - 1
         df['EMA_21_dist'] = (c / df['ema21'].values) - 1
         df['EMA_50_dist'] = (c / df['ema50'].values) - 1
         df['EMA_200_dist'] = (c / df['ema200'].values) - 1
         df['VWAP_dist'] = (c / df['vwap'].values) - 1
-        df['ATR_pct'] = df['ATR'] / (c + 1e-9)
+        df['ATR_pct'] = (df['ATR'] / (c + 1e-9)) * 100 # In Percent
         
         if timeframe == '1d': df['Trend_Strong'] = np.where(df['ADX'] > 25, 1.0, 0.0)
         
         df['vol_z'] = _z_roll_np(df['volume'].values, 20)
+        # Relative Volume (Important for Explosive State)
         df['rel_vol'] = df['volume'] / (df['volume'].rolling(50).mean() + 1e-9)
         df['log_ret'] = np.concatenate([[0], np.diff(np.log(c + 1e-9))])
         
@@ -237,43 +240,63 @@ class HeavyDutyBacktester:
         map_1h = get_map('1h'); map_15m = get_map('15m')
 
         # ============================================================
-        # ✅ 2. NEW L1 LOGIC (Vectorized Implementation of DataManager)
+        # 🛡️ 1. MARKET KILL-SWITCH (Proxy based on Local Coin Data)
+        # ============================================================
+        # If the coin itself is dead/choppy, we assume the market for it is dead.
+        # Logic: High Chop + Low Volatility + Low ADX = DEAD
+        h1_chop = numpy_htf['1h']['CHOP'][map_1h]
+        h1_adx = numpy_htf['1h']['ADX'][map_1h]
+        h1_atr_pct = numpy_htf['1h']['ATR_pct'][map_1h]
+
+        # DEAD Condition: 
+        # (Chop > 61.8) OR (ATR% < 0.3 AND ADX < 20)
+        is_market_dead = (h1_chop > 61.8) | ((h1_atr_pct < 0.3) & (h1_adx < 20))
+        market_status = np.where(is_market_dead, 0, 1) # 0 = DEAD, 1 = OK
+
         # ============================================================
-        # Logic is applied on 1H frame, mapped to 1m
+        # 💎 2. COIN STATES CLASSIFICATION (Vectorized)
+        # ============================================================
+        # States: 0=TRASH, 1=ACCUMULATION, 2=SAFE_TREND, 3=EXPLOSIVE
+        
         h1_rsi = numpy_htf['1h']['RSI'][map_1h]
         h1_close = numpy_htf['1h']['close'][map_1h]
+        h1_bbw = numpy_htf['1h']['bb_width'][map_1h]
+        h1_upper = numpy_htf['1h']['upper_bb'][map_1h]
         h1_ema20 = numpy_htf['1h']['ema20'][map_1h]
         h1_ema50 = numpy_htf['1h']['ema50'][map_1h]
         h1_ema200 = numpy_htf['1h']['ema200'][map_1h]
-        h1_lower_bb = numpy_htf['1h']['lower_bb'][map_1h]
-        h1_upper_bb = numpy_htf['1h']['upper_bb'][map_1h]
-        h1_bb_width = numpy_htf['1h']['bb_width'][map_1h]
+        h1_rel_vol = numpy_htf['1h']['rel_vol'][map_1h]
+
+        # Initialize as TRASH (0)
+        coin_state = np.zeros(len(arr_ts_1m), dtype=np.int8)
+
+        # A. TRASH (0) - Defined implicitly by what is NOT the others, 
+        # plus explicit check for very low liquidity
+        is_trash_vol = (h1_rel_vol < 0.5) | (h1_atr_pct < 0.2)
         
-        # TYPE 1: SAFE_BOTTOM
-        # (rsi < 45) & (close <= lower_bb * 1.05) & (dist_from_ema50 > 0.015)
-        dist_from_ema50 = (h1_ema50 - h1_close) / h1_ema50
-        mask_safe_bottom = (h1_rsi < 45) & (h1_close <= h1_lower_bb * 1.05) & (dist_from_ema50 > 0.015)
+        # B. ACCUMULATION (1)
+        # Squeeze (Low BBW) + RSI Neutral
+        mask_acc = (h1_bbw < 0.15) & (h1_rsi >= 40) & (h1_rsi <= 60)
         
-        # TYPE 2: ACCUMULATION_SQUEEZE
-        # (45 <= rsi <= 60) & (bb_width < 0.12) & (close > ema20 * 0.995)
-        mask_acc_squeeze = (h1_rsi >= 45) & (h1_rsi <= 60) & (h1_bb_width < 0.12) & (h1_close > h1_ema20 * 0.995)
+        # C. SAFE_TREND (2)
+        # ADX Strong + EMA Stacked + Healthy RSI
+        mask_safe = (h1_adx > 25) & (h1_ema20 > h1_ema50) & (h1_ema50 > h1_ema200) & (h1_rsi > 50) & (h1_rsi < 75)
         
-        # TYPE 3: MOMENTUM_LAUNCH
-        # (60 < rsi < 80) & (close > ema50) & (close > ema200) & (dist_to_upper < 0.08)
-        dist_to_upper = (h1_upper_bb - h1_close) / h1_close
-        mask_mom_launch = (h1_rsi > 60) & (h1_rsi < 80) & (h1_close > h1_ema50) & (h1_close > h1_ema200) & (dist_to_upper < 0.08)
+        # D. EXPLOSIVE (3)
+        # High RSI + Breakout + Volume Spike
+        mask_exp = (h1_rsi > 65) & (h1_close > h1_upper) & (h1_rel_vol > 1.5)
+
+        # Apply Priorities (Explosive > Safe > Acc > Trash)
+        coin_state[mask_acc] = 1
+        coin_state[mask_safe] = 2
+        coin_state[mask_exp] = 3
+        coin_state[is_trash_vol] = 0 # Force trash if liquidity is bad
 
-        # Combine Masks (Any Valid L1)
-        valid_l1_mask = mask_safe_bottom | mask_acc_squeeze | mask_mom_launch
-        
         # ============================================================
-        # ✅ 4. GOVERNANCE PROXY (Vectorized Scoring)
+        # ✅ GOVERNANCE PROXY (Vectorized Scoring)
         # ============================================================
-        # Mimics governance_engine.py logic (Simplified for speed)
         gov_points = np.zeros(len(arr_ts_1m), dtype=np.float32)
         
-        # Trend (30%)
-        # EMA9 > EMA21 on 15m
         m15_ema9 = numpy_htf['15m']['ema9'][map_15m]
         m15_ema21 = numpy_htf['15m']['ema21'][map_15m]
         m15_ema50 = numpy_htf['15m']['ema50'][map_15m]
@@ -283,7 +306,6 @@ class HeavyDutyBacktester:
         gov_points += np.where(m15_ema21 > m15_ema50, 10.0, 0.0)
         gov_points += np.where(m15_close > numpy_htf['15m']['ema200'][map_15m], 5.0, 0.0)
         
-        # Momentum (30%)
         m15_rsi = numpy_htf['15m']['RSI'][map_15m]
         m15_macd = numpy_htf['15m']['MACD'][map_15m]
         m15_macd_s = numpy_htf['15m']['MACD_s'][map_15m]
@@ -291,16 +313,11 @@ class HeavyDutyBacktester:
         gov_points += np.where((m15_rsi > 45) & (m15_rsi < 70), 15.0, 0.0)
         gov_points += np.where(m15_macd > m15_macd_s, 15.0, 0.0)
         
-        # Volatility & Volume (20%)
         m15_bbw = numpy_htf['15m']['bb_width'][map_15m]
-        gov_points += np.where((m15_bbw > 0.02) & (m15_bbw < 0.15), 10.0, 0.0) # Not too tight, not too wide
+        gov_points += np.where((m15_bbw > 0.02) & (m15_bbw < 0.15), 10.0, 0.0)
         gov_points += np.where(numpy_htf['15m']['rel_vol'][map_15m] > 1.0, 10.0, 0.0)
-        
-        # Structure (20%)
-        # Price above VWAP
         gov_points += np.where(m15_close > numpy_htf['15m']['vwap'][map_15m], 20.0, 0.0)
         
-        # Final Governance Score (0-100)
         gov_scores_final = np.clip(gov_points, 0, 100)
         
         # ============================================================
@@ -314,7 +331,6 @@ class HeavyDutyBacktester:
 
         global_titan_scores = np.full(len(arr_ts_1m), 0.5, dtype=np.float32)
         if titan_model:
-            # (Titan prediction logic kept same as original for stability)
             titan_cols = [
                 '5m_open', '5m_high', '5m_low', '5m_close', '5m_volume', '5m_RSI', '5m_MACD', '5m_MACD_h', 
                 '5m_CCI', '5m_ADX', '5m_EMA_9_dist', '5m_EMA_21_dist', '5m_EMA_50_dist', '5m_EMA_200_dist', 
@@ -326,7 +342,6 @@ class HeavyDutyBacktester:
                 '1d_RSI', '1d_EMA_200_dist', '1d_Trend_Strong'
             ]
             try:
-                # Need map_5m and map_4h here strictly for Titan
                 def get_map_local(tf):
                     if tf not in numpy_htf: return np.zeros(len(arr_ts_1m), dtype=int)
                     return np.clip(np.searchsorted(numpy_htf[tf]['timestamp'], arr_ts_1m), 0, len(numpy_htf[tf]['timestamp']) - 1)
@@ -348,7 +363,6 @@ class HeavyDutyBacktester:
         global_oracle_scores = np.full(len(arr_ts_1m), 0.5, dtype=np.float32)
         if oracle_dir:
             try:
-                # Need map_4h for Oracle too
                 map_4h = locals().get('map_4h', get_map('4h'))
                 o_vecs = []
                 for col in oracle_cols:
@@ -378,19 +392,16 @@ class HeavyDutyBacktester:
                 global_sniper_scores = _revive_score_distribution(np.mean(preds, axis=0))
             except: pass
 
-        # ✅ 3. DISABLE GUARDIANS IN BACKTEST (Requested Update)
-        # We skip Hydra and Legacy V2 predictions to save time and because user requested.
-        
-        # Filter (Refined with new L1)
-        # Keep candles where L1 is Valid OR Scores are very high
-        is_candidate_mask = valid_l1_mask | ((global_titan_scores > 0.6) & (global_oracle_scores > 0.6))
+        # Filter: Must NOT be TRASH, Must be Market OK, Must have decent scores
+        is_candidate_mask = (coin_state > 0) & (market_status == 1) & \
+                            ((global_titan_scores > 0.6) & (global_oracle_scores > 0.6))
         
         candidate_indices = np.where(is_candidate_mask)[0]
         end_limit = len(arr_ts_1m) - 60
         candidate_indices = candidate_indices[candidate_indices < end_limit]
         candidate_indices = candidate_indices[candidate_indices >= 500]
 
-        print(f"     🌪️ Final List: {len(candidate_indices)} candidates ready for testing.", flush=True)
+        print(f"     🌪️ Final List: {len(candidate_indices)} candidates (Trash/Dead filtered).", flush=True)
 
         ai_results = pd.DataFrame({
             'timestamp': arr_ts_1m[candidate_indices],
@@ -399,8 +410,9 @@ class HeavyDutyBacktester:
             'real_titan': global_titan_scores[candidate_indices],
             'oracle_conf': global_oracle_scores[candidate_indices],
             'sniper_score': global_sniper_scores[candidate_indices],
-            'gov_score': gov_scores_final[candidate_indices], # ✅ New Gov Score
-            'l1_valid': valid_l1_mask[candidate_indices].astype(int) # ✅ New L1 Flag
+            'gov_score': gov_scores_final[candidate_indices],
+            'coin_state': coin_state[candidate_indices], # 1=ACC, 2=SAFE, 3=EXP
+            'market_ok': market_status[candidate_indices]
         })
 
         dt = time.time() - t0
@@ -431,7 +443,7 @@ class HeavyDutyBacktester:
 
     @staticmethod
     def _worker_optimize(combinations_batch, scores_files, initial_capital, fees_pct, max_slots):
-        """🚀 HYPER-SPEED JUMP LOGIC (NO GUARDIANS, ADDED GOV)"""
+        """🚀 HYPER-SPEED JUMP LOGIC (Filtering TRASH + DEAD MARKETS)"""
         print(f"     ⏳ [System] Loading {len(scores_files)} datasets...", flush=True)
         data = []
         for f in scores_files:
@@ -449,16 +461,19 @@ class HeavyDutyBacktester:
         oracle = df['oracle_conf'].values
         sniper = df['sniper_score'].values
         titan = df['real_titan'].values
-        gov_s = df['gov_score'].values # ✅ Loaded Gov Score
-        l1_v = df['l1_valid'].values   # ✅ Loaded L1 Valid Flag
+        gov_s = df['gov_score'].values
+        c_state = df['coin_state'].values # 0=TRASH, 1=ACC, 2=SAFE, 3=EXP
+        m_ok = df['market_ok'].values     # 1=OK, 0=DEAD
         
         N = len(ts)
         print(f"     🚀 [System] Testing {len(combinations_batch)} configs on {N} candidates...", flush=True)
         
         res = []
         for cfg in combinations_batch:
-            # ✅ Updated Entry Mask: Requires Valid L1 & Gov Score
-            entry_mask = (l1_v == 1) & \
+            # ✅ Updated Entry Mask: Kill-Switch + Coin State
+            # Must be Market OK (1) AND Coin State > 0 (Not TRASH)
+            entry_mask = (m_ok == 1) & \
+                         (c_state > 0) & \
                          (gov_s >= cfg['GOV_SCORE']) & \
                          (oracle >= cfg['ORACLE']) & \
                          (sniper >= cfg['SNIPER']) & \
@@ -466,8 +481,7 @@ class HeavyDutyBacktester:
             
             valid_entry_indices = np.where(entry_mask)[0]
             
-            # Simulation State
-            pos = {} # sym_id -> (entry_price, size)
+            pos = {} 
             bal = float(initial_capital)
             alloc = 0.0
             log = []
@@ -475,12 +489,11 @@ class HeavyDutyBacktester:
             for i in range(N):
                 s = sym_id[i]; p = float(close[i])
                 
-                # A. Check Exits (Standard TP/SL Logic only)
+                # A. Check Exits 
                 if s in pos:
                     entry_p, size_val = pos[s]
                     pnl = (p - entry_p) / entry_p
                     
-                    # Exit Rules (Standard Backtest)
                     if (pnl > 0.04) or (pnl < -0.02):
                         realized = pnl - (fees_pct * 2)
                         bal += size_val * (1.0 + realized)
@@ -540,13 +553,13 @@ class HeavyDutyBacktester:
         
         mapped_config = {
             'w_titan': best['config']['TITAN'],
-            'w_struct': 0.3, # Fixed as removed from grid
-            'thresh': 50.0,  # Fixed L1 Logic
+            'w_struct': 0.3, 
+            'thresh': 50.0,  
             'oracle_thresh': best['config']['ORACLE'],
             'sniper_thresh': best['config']['SNIPER'],
             'gov_thresh': best['config']['GOV_SCORE'],
-            'hydra_thresh': 0.85, # Default Safe
-            'legacy_thresh': 0.95 # Default Safe
+            'hydra_thresh': 0.85, 
+            'legacy_thresh': 0.95 
         }
 
         # Diagnosis
@@ -585,14 +598,9 @@ async def run_strategic_optimization_task():
         hub = AdaptiveHub(r2); await hub.initialize()
         opt = HeavyDutyBacktester(dm, proc)
         
-        # Scenarios now just set the regime target for saving, dates handled internally
-        scenarios = [
-            {"regime": "RANGE"}, 
-            # Add more regimes if needed to run consecutively
-        ]
+        scenarios = [{"regime": "RANGE"}]
         
         for s in scenarios:
-            # Dates are now handled by LOOKBACK_DAYS in constructor by default
             best_cfg, best_stats = await opt.run_optimization(s["regime"])
             if best_cfg: hub.submit_challenger(s["regime"], best_cfg, best_stats)