Update backtest_engine.py

backtest_engine.py

@@ -104,88 +104,93 @@ def _transform_window_for_pattern(df_window):
 
 def calculate_sniper_features_exact(df):
     """
-    Sniper Features Calculation - With AUTO-NORMALIZATION Patch.
-    Converts price-dependent features to percentages to fix Scale Mismatch.
+    Sniper Features Calculation - TRUE UNIVERSAL MODE (Z-SCORE).
+    Converts structural features to Z-Scores to bypass scale issues.
     """
     # 1. Standard Features
     d = df.copy()
     c = d['close']; h = d['high']; l = d['low']; v = d['volume']; o = d['open']
     
-    # Basic Returns
+    # --- Helper: Rolling Z-Score ---
+    def _z_roll(x, w=200): # Window 200 is standard for regime detection
+        r = x.rolling(w).mean()
+        s = x.rolling(w).std().replace(0, np.nan)
+        return ((x - r) / s).fillna(0)
+
+    # Basic Returns (Keep as percentages, trees handle these well)
     d['return_1m'] = c.pct_change(1).fillna(0)
     d['return_3m'] = c.pct_change(3).fillna(0)
     d['return_5m'] = c.pct_change(5).fillna(0)
     d['return_15m'] = c.pct_change(15).fillna(0)
 
-    # Technicals
+    # Technicals (RSI is bounded 0-100, usually safe)
     d['rsi_14'] = ta.rsi(c, length=14).fillna(50)
     
     ema_9 = ta.ema(c, length=9).fillna(c)
     ema_21 = ta.ema(c, length=21).fillna(c)
     
+    # Slopes/Distances -> Normalized
     d['ema_9_slope'] = ((ema_9 - ema_9.shift(1)) / ema_9.shift(1).replace(0, np.nan)).fillna(0)
     d['ema_21_dist'] = ((c - ema_21) / ema_21.replace(0, np.nan)).fillna(0)
     
-    # --- GEM-ARCHITECT PATCH: Normalize Price-Dependent Features ---
-    # ATR (Normalized by Close)
+    # --- TRANSFORM 1: ATR (Vol) -> Z-Score ---
+    # Instead of raw value or %, check if Volatility is spiking relative to history
     atr_raw = ta.atr(h, l, c, length=100).fillna(0)
-    d['atr'] = (atr_raw / c).fillna(0)  # Now it's a percentage (e.g., 0.002 instead of 0.25)
-    
-    # Volume Z-Score (Already normalized by z-score logic)
-    def _z_roll(x, w=50):
-        r = x.rolling(w).mean()
-        s = x.rolling(w).std().replace(0, np.nan)
-        return ((x - r) / s).fillna(0)
+    d['atr'] = _z_roll(atr_raw, 500) 
     
+    # Volume Z-Score
     d['vol_zscore_50'] = _z_roll(v, 50)
     
-    # Candle Geometry (Normalized)
-    rng = (h - l)
-    d['candle_range'] = (rng / c).fillna(0) # Normalized
-    d['close_pos_in_range'] = ((c - l) / rng.replace(0, 1e-9)).fillna(0.5)
+    # Candle Geometry
+    rng = (h - l).replace(0, 1e-9)
+    d['candle_range'] = _z_roll(rng, 500) # Is this candle huge relative to history?
+    d['close_pos_in_range'] = ((c - l) / rng).fillna(0.5)
 
-    # 2. Liquidity Proxies
-    # Amihud (Ratio of %Ret to $Vol -> inherently somewhat scale-invariant but sensitive)
+    # --- TRANSFORM 2: Liquidity Proxies -> Z-Score ---
+    # This fixes the Amihud 1e-8 issue completely.
+    
+    # Amihud
     d['dollar_vol'] = c * v
-    d['amihud'] = (d['return_1m'].abs() / d['dollar_vol'].replace(0, np.nan)).fillna(0)
+    amihud_raw = (d['return_1m'].abs() / d['dollar_vol'].replace(0, np.nan)).fillna(0)
+    d['amihud'] = _z_roll(amihud_raw, 500)
     
-    # Roll Spread (Normalized)
+    # Roll Spread
     dp = c.diff()
     roll_cov = dp.rolling(64).cov(dp.shift(1)).fillna(0)
     roll_spread_raw = (2 * np.sqrt(np.maximum(0, -roll_cov)))
-    d['roll_spread'] = (roll_spread_raw / c).fillna(0) # Normalized
+    d['roll_spread'] = _z_roll(roll_spread_raw, 500)
     
-    # OFI (Volume based -> Normalized by Z-Score later or used raw if model expects raw volume flow)
-    # Usually OFI is used as a signal direction, kept raw or z-scored. 
-    # Let's keep raw here as it's volume-based, not price-based.
+    # OFI (Order Flow)
     sign = np.sign(c.diff()).fillna(0)
     d['signed_vol'] = sign * v
-    d['ofi'] = d['signed_vol'].rolling(30).sum().fillna(0)
+    ofi_raw = d['signed_vol'].rolling(30).sum().fillna(0)
+    d['ofi'] = _z_roll(ofi_raw, 500)
     
-    # VPIN (Ratio -> Unitless -> Safe)
+    # VPIN
     buy_vol = (sign > 0) * v
     sell_vol = (sign < 0) * v
     imb = (buy_vol.rolling(60).sum() - sell_vol.rolling(60).sum()).abs()
     tot = v.rolling(60).sum().replace(0, np.nan)
-    d['vpin'] = (imb / tot).fillna(0)
+    d['vpin'] = (imb / tot).fillna(0) # VPIN is a ratio (0-1), usually safe without Z-score, but let's keep it raw
     
-    # Garman-Klass (Log returns -> Unitless -> Safe)
-    d['rv_gk'] = ((np.log(h / l)**2) / 2) - ((2 * np.log(2) - 1) * (np.log(c / o)**2))
-    d['rv_gk'] = d['rv_gk'].fillna(0)
+    # Garman-Klass Volatility
+    rv_gk_raw = ((np.log(h / l)**2) / 2) - ((2 * np.log(2) - 1) * (np.log(c / o)**2))
+    d['rv_gk'] = _z_roll(rv_gk_raw.fillna(0), 500)
     
-    # VWAP Deviation (Normalized)
+    # VWAP Deviation
     vwap_win = 20
     vwap = (d['dollar_vol'].rolling(vwap_win).sum() / v.rolling(vwap_win).sum().replace(0, np.nan)).fillna(c)
-    d['vwap_dev'] = ((c - vwap) / c).fillna(0) # Normalized
+    d['vwap_dev'] = _z_roll((c - vwap), 500)
     
-    # Liquidity Score (Composite - Z-scores handle scaling automatically)
+    # Liquidity Score (Composite - Already using internal Z-scores in logic, but let's re-calc)
+    # Note: We use the already Z-scored columns now where possible
     d['L_score'] = (
-        _z_roll(v, 500) + 
-        _z_roll(1 / (d['amihud'] + 1e-12), 500) + 
-        _z_roll(-d['roll_spread'], 500) + 
-        _z_roll(-d['rv_gk'].abs(), 500) + 
-        _z_roll(-d['vwap_dev'].abs(), 500) + 
-        _z_roll(d['ofi'], 500)
+        d['vol_zscore_50'] + 
+        (-d['amihud']) +  # Low illiquidity is good
+        (-d['roll_spread']) + 
+        (-d['rv_gk'].abs()) + 
+        (-d['vwap_dev'].abs()) + 
+        d['ofi']
     ).fillna(0)
     
     return sanitize_features(d)