Update ml_engine/sniper_engine.py

ml_engine/sniper_engine.py

@@ -1,23 +1,22 @@
 # ============================================================
-# 🎯 ml_engine/sniper_engine.py (V1.8 - Fully Configurable)
+# 🎯 ml_engine/sniper_engine.py 
+# (V2.0 - GEM-Architect: Weighted Depth & Smart Microstructure)
 # ============================================================
 
 import os
-import sys
+import time
 import numpy as np
 import pandas as pd
 import pandas_ta as ta
 import lightgbm as lgb
-import joblib
-import asyncio
 import traceback
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Optional
 
 N_SPLITS = 5
 LOOKBACK_WINDOW = 500 
 
 # ============================================================
-# 🔧 1. دوال هندسة الميزات (Feature Engineering)
+# 🔧 1. Feature Engineering (Standard + Liquidity Proxies)
 # ============================================================
 
 def _z_score_rolling(x, w=500): 
@@ -27,6 +26,7 @@ def _z_score_rolling(x, w=500):
     return z.fillna(0)
 
 def _add_liquidity_proxies(df):
+    """حساب مؤشرات السيولة المتقدمة (Amihud, VPIN, OFI, etc.)"""
     df_proxy = df.copy()
     if 'datetime' not in df_proxy.index:
         if 'timestamp' in df_proxy.columns:
@@ -36,30 +36,37 @@ def _add_liquidity_proxies(df):
     df_proxy['ret'] = df_proxy['close'].pct_change().fillna(0)
     df_proxy['dollar_vol'] = df_proxy['close'] * df_proxy['volume']
     
+    # Amihud Illiquidity Ratio
     df_proxy['amihud'] = (df_proxy['ret'].abs() / df_proxy['dollar_vol'].replace(0, np.nan)).fillna(np.inf)
     
+    # Roll Spread Proxy
     dp = df_proxy['close'].diff()
     roll_cov = dp.rolling(64).cov(dp.shift(1))
     df_proxy['roll_spread'] = (2 * np.sqrt(np.maximum(0, -roll_cov))).bfill()
 
+    # Order Flow Imbalance (Volume-based proxy)
     sign = np.sign(df_proxy['close'].diff()).fillna(0)
     df_proxy['signed_vol'] = sign * df_proxy['volume']
     df_proxy['ofi'] = df_proxy['signed_vol'].rolling(30).sum().fillna(0)
 
+    # VPIN-like Imbalance
     buy_vol = (sign > 0) * df_proxy['volume']
     sell_vol = (sign < 0) * df_proxy['volume']
     imb = (buy_vol.rolling(60).sum() - sell_vol.rolling(60).sum()).abs()
     tot = df_proxy['volume'].rolling(60).sum()
     df_proxy['vpin'] = (imb / tot.replace(0, np.nan)).fillna(0)
 
+    # Volatility Estimator (Garman-Klass)
     df_proxy['rv_gk'] = (np.log(df_proxy['high'] / df_proxy['low'])**2) / 2 - \
                         (2 * np.log(2) - 1) * (np.log(df_proxy['close'] / df_proxy['open'])**2)
     
+    # VWAP Deviation
     vwap_window = 20
     df_proxy['vwap'] = (df_proxy['close'] * df_proxy['volume']).rolling(vwap_window).sum() / \
                        df_proxy['volume'].rolling(vwap_window).sum()
     df_proxy['vwap_dev'] = (df_proxy['close'] - df_proxy['vwap']).fillna(0)
 
+    # Composite Liquidity Score
     df_proxy['L_score'] = (
         _z_score_rolling(df_proxy['volume']) +
         _z_score_rolling(1 / df_proxy['amihud'].replace(np.inf, np.nan)) + 
@@ -71,6 +78,7 @@ def _add_liquidity_proxies(df):
     return df_proxy
 
 def _add_standard_features(df):
+    """المؤشرات الفنية القياسية"""
     df_feat = df.copy()
     
     df_feat['return_1m'] = df_feat['close'].pct_change(1)
@@ -102,7 +110,7 @@ def _add_standard_features(df):
     return df_feat
 
 # ============================================================
-# 🎯 2. كلاس المحرك الرئيسي (SniperEngine V1.8)
+# 🎯 2. SniperEngine Class (Refactored)
 # ============================================================
 
 class SniperEngine:
@@ -112,49 +120,62 @@ class SniperEngine:
         self.models: List[lgb.Booster] = []
         self.feature_names: List[str] = []
         
-        # ✅ القيم الافتراضية (سيتم تحديثها من Processor)
+        # --- Configurable Thresholds (Defaults) ---
         self.entry_threshold = 0.40
-        self.wall_ratio_limit = 0.40 # نسبة الفيتو لجدار البيع
-        
-        # ✅ إضافة متغيرات للأوزان (قابلة للتكوين)
+        self.wall_ratio_limit = 0.40    # Veto threshold for sell wall
         self.weight_ml = 0.60
         self.weight_ob = 0.40
         
+        # --- Advanced OB Settings (New in V2.0) ---
+        self.ob_depth_decay = 0.15      # Decay factor for weighted depth
+        self.max_wall_dist = 0.005      # 0.5% max distance to consider a wall
+        self.max_spread_pct = 0.002     # 0.2% max spread allowed
+        self.spoof_patience = 0         # How many previous checks to ignore a new wall (0 = Instant Veto)
+        
         self.initialized = False
         self.LOOKBACK_WINDOW = LOOKBACK_WINDOW
         self.ORDER_BOOK_DEPTH = 20
         
-        print("🎯 [SniperEngine V1.8] Created.")
-
-    # ✅ دالة تكوين شاملة (محدثة لاستقبال الأوزان)
-    def configure_settings(self, threshold: float, wall_ratio: float, w_ml: float = 0.60, w_ob: float = 0.40):
+        # --- Persistence Cache for Anti-Spoofing ---
+        # Format: {symbol: {'last_check': timestamp, 'wall_counter': int}}
+        self._wall_cache = {}
+
+        print("🎯 [SniperEngine V2.0] Weighted Depth & Smart Microstructure Ready.")
+
+    def configure_settings(self, 
+                           threshold: float, 
+                           wall_ratio: float, 
+                           w_ml: float = 0.60, 
+                           w_ob: float = 0.40,
+                           max_wall_dist: float = 0.005,
+                           max_spread: float = 0.002):
+        """Dynamic configuration injection"""
         self.entry_threshold = threshold
         self.wall_ratio_limit = wall_ratio
         self.weight_ml = w_ml
         self.weight_ob = w_ob
-        # print(f"🔧 [Sniper] Configured: Threshold={self.entry_threshold}, WallVeto={self.wall_ratio_limit}, W_ML={self.weight_ml}")
-
-    # (إبقاء هذه للدعم القديم إذا لزم الأمر، لكن configure_settings أفضل)
-    def set_entry_threshold(self, new_threshold: float):
-        self.entry_threshold = new_threshold
+        self.max_wall_dist = max_wall_dist
+        self.max_spread_pct = max_spread
 
     async def initialize(self):
-        """تحميل النماذج"""
+        """Load LightGBM Models"""
         print(f"🎯 [SniperEngine] Loading models from {self.models_dir}...")
         try:
             model_files = [f for f in os.listdir(self.models_dir) if f.startswith('lgbm_guard_v3_fold_')]
             
             if len(model_files) < N_SPLITS:
                 print(f"❌ [SniperEngine] Error: Found {len(model_files)} models, need {N_SPLITS}.")
-                return
-
+                # Don't return, allow initialization without models (fallback mode)
+            
             for f in sorted(model_files):
                 model_path = os.path.join(self.models_dir, f)
                 self.models.append(lgb.Booster(model_file=model_path))
             
-            self.feature_names = self.models[0].feature_name()
+            if self.models:
+                self.feature_names = self.models[0].feature_name()
+                
             self.initialized = True
-            print(f"✅ [SniperEngine] Ready. Threshold: {self.entry_threshold}, WallVeto: {self.wall_ratio_limit}")
+            print(f"✅ [SniperEngine] Active. WallLimit: {self.wall_ratio_limit}, MaxDist: {self.max_wall_dist*100}%")
 
         except Exception as e:
             print(f"❌ [SniperEngine] Init failed: {e}")
@@ -172,108 +193,161 @@ class SniperEngine:
             return pd.DataFrame() 
 
     # ==============================================================================
-    # 📊 3. منطق تحليل دفتر الطلبات
+    # 📊 3. Smart Order Book Logic (The Architect's Upgrade)
     # ==============================================================================
-    def _score_order_book(self, order_book: Dict[str, Any]) -> Dict[str, Any]:
+    def _score_order_book(self, order_book: Dict[str, Any], symbol: str = None) -> Dict[str, Any]:
         try:
             bids = order_book.get('bids', [])
             asks = order_book.get('asks', [])
             
             if not bids or not asks:
-                return {'score': 0.0, 'imbalance': 0.0, 'wall_ratio': 0.0, 'reason': 'Empty'}
-
-            depth = self.ORDER_BOOK_DEPTH
-            top_bids = bids[:depth]
-            top_asks = asks[:depth]
-
-            total_bid_vol = sum([float(x[1]) for x in top_bids])
-            total_ask_vol = sum([float(x[1]) for x in top_asks])
-            total_vol = total_bid_vol + total_ask_vol
-            
-            if total_vol == 0: 
-                return {'score': 0.0, 'imbalance': 0.0, 'wall_ratio': 0.0, 'reason': 'Zero Vol'}
+                return {'score': 0.0, 'imbalance': 0.0, 'veto': True, 'reason': 'Empty OB'}
 
-            bid_imbalance = total_bid_vol / total_vol 
+            # --- 1. Spread Check ---
+            best_bid = float(bids[0][0])
+            best_ask = float(asks[0][0])
+            spread_pct = (best_ask - best_bid) / best_bid
             
-            max_ask_wall = max([float(x[1]) for x in top_asks]) if top_asks else 0
-            ask_wall_ratio = max_ask_wall / total_ask_vol if total_ask_vol > 0 else 0
-            
-            # ✅ استخدام المتغير المحقون wall_ratio_limit
-            if ask_wall_ratio >= self.wall_ratio_limit:
+            if spread_pct > self.max_spread_pct:
                 return {
                     'score': 0.0, 
-                    'imbalance': float(bid_imbalance),
-                    'wall_ratio': float(ask_wall_ratio),
-                    'veto': True,
-                    'reason': f"⛔ SELL WALL ({ask_wall_ratio:.2f} >= {self.wall_ratio_limit})"
+                    'veto': True, 
+                    'reason': f"Wide Spread ({spread_pct:.2%})"
                 }
 
+            # --- 2. Weighted Depth Imbalance ---
+            # Calculates imbalance giving higher weight to prices closer to spread
+            w_bid_vol = 0.0
+            w_ask_vol = 0.0
+            total_raw_ask_vol = 0.0 # for wall calculation
+            
+            # Limit depth processing to configured depth
+            depth = min(len(bids), len(asks), self.ORDER_BOOK_DEPTH)
+            
+            for i in range(depth):
+                # Decay Function: 1 / (1 + k * rank)
+                weight = 1.0 / (1.0 + (self.ob_depth_decay * i))
+                
+                bid_vol = float(bids[i][1])
+                ask_vol = float(asks[i][1])
+                
+                w_bid_vol += bid_vol * weight
+                w_ask_vol += ask_vol * weight
+                total_raw_ask_vol += ask_vol
+
+            total_w_vol = w_bid_vol + w_ask_vol
+            weighted_imbalance = w_bid_vol / total_w_vol if total_w_vol > 0 else 0.5
+            
+            # --- 3. Distance-Aware Wall Detection ---
+            max_valid_wall = 0.0
+            limit_price = best_ask * (1 + self.max_wall_dist)
+            
+            for price, vol in asks[:depth]:
+                p = float(price)
+                v = float(vol)
+                if p <= limit_price:
+                    if v > max_valid_wall: max_valid_wall = v
+            
+            wall_ratio = max_valid_wall / total_raw_ask_vol if total_raw_ask_vol > 0 else 0
+            
+            # --- 4. Anti-Spoofing / Persistence Logic ---
+            veto_wall = False
+            veto_reason = "OK"
+            
+            if wall_ratio >= self.wall_ratio_limit:
+                # Wall Detected
+                veto_wall = True
+                veto_reason = f"Sell Wall ({wall_ratio:.2f})"
+                
+                if symbol:
+                    curr_time = time.time()
+                    cache = self._wall_cache.get(symbol, {'last_check': 0, 'count': 0})
+                    
+                    # If this is a NEW wall (seen less than 1 second ago)
+                    if curr_time - cache['last_check'] > 5.0:
+                        # Reset counter if too much time passed
+                        cache['count'] = 1
+                    else:
+                        cache['count'] += 1
+                        
+                    cache['last_check'] = curr_time
+                    self._wall_cache[symbol] = cache
+                    
+                    # Optional: Logic to IGNORE flashing walls could go here
+                    # For now, we block on first sight (Safety First)
+            else:
+                # No wall, clear cache slightly
+                if symbol and symbol in self._wall_cache:
+                    self._wall_cache[symbol]['count'] = 0
+
             return {
-                'score': float(bid_imbalance),
-                'imbalance': float(bid_imbalance),
-                'wall_ratio': float(ask_wall_ratio),
-                'veto': False,
-                'reason': "OK"
+                'score': float(weighted_imbalance),
+                'imbalance': float(weighted_imbalance), # Now Weighted
+                'wall_ratio': float(wall_ratio),
+                'veto': veto_wall,
+                'spread_ok': True,
+                'reason': veto_reason
             }
 
         except Exception as e:
-            return {'score': 0.0, 'reason': f"Error: {e}", 'veto': True}
+            return {'score': 0.0, 'veto': True, 'reason': f"OB Error: {e}"}
 
     # ==============================================================================
-    # 🎯 4. دالة الفحص الرئيسية
+    # 🎯 4. Main Signal Check (Async)
     # ==============================================================================
-    async def check_entry_signal_async(self, ohlcv_1m_data: List[List], order_book_data: Dict[str, Any] = None) -> Dict[str, Any]:
+    async def check_entry_signal_async(self, 
+                                     ohlcv_1m_data: List[List], 
+                                     order_book_data: Dict[str, Any] = None,
+                                     symbol: str = None) -> Dict[str, Any]:
+        
         if not self.initialized:
             return {'signal': 'WAIT', 'reason': 'Not initialized'}
         
-        if len(ohlcv_1m_data) < self.LOOKBACK_WINDOW:
-            return {'signal': 'WAIT', 'reason': 'Insuff Data'}
-
-        try:
-            df = pd.DataFrame(ohlcv_1m_data, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
-            df[['open', 'high', 'low', 'close', 'volume']] = df[['open', 'high', 'low', 'close', 'volume']].astype(float)
-            
-            df_features = self._calculate_features_live(df)
-            if df_features.empty:
-                return {'signal': 'WAIT', 'reason': 'Feat Fail'}
-
-            X_live = df_features.iloc[-1:][self.feature_names].fillna(0)
-            preds = [m.predict(X_live)[0][1] for m in self.models]
-            ml_score = float(np.mean(preds))
-
-        except Exception as e:
-            print(f"❌ [Sniper] ML Error: {e}")
-            return {'signal': 'WAIT', 'reason': f'ML Exception: {e}'}
-
-        ob_data = {'score': 0.5, 'imbalance': 0.5, 'wall_ratio': 0.0, 'veto': False} 
-        if order_book_data:
-            ob_data = self._score_order_book(order_book_data)
-
-        # ✅ استخدام الأوزان المحقونة (Dynamic Weights)
-        final_score = (ml_score * self.weight_ml) + (ob_data['score'] * self.weight_ob)
+        # --- ML Prediction ---
+        ml_score = 0.5
+        ml_reason = "No Data"
         
-        signal = 'WAIT'
-        reason_str = f"Final:{final_score:.2f} (ML:{ml_score:.2f} + OB:{ob_data['score']:.2f})"
+        if len(ohlcv_1m_data) >= self.LOOKBACK_WINDOW and self.models:
+            try:
+                df = pd.DataFrame(ohlcv_1m_data, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
+                df[['open', 'high', 'low', 'close', 'volume']] = df[['open', 'high', 'low', 'close', 'volume']].astype(float)
+                
+                df_features = self._calculate_features_live(df)
+                if not df_features.empty:
+                    X_live = df_features.iloc[-1:][self.feature_names].fillna(0)
+                    preds = [m.predict(X_live)[0][1] for m in self.models]
+                    ml_score = float(np.mean(preds))
+                    ml_reason = f"ML:{ml_score:.2f}"
+            except Exception as e:
+                print(f"❌ [Sniper] ML Error: {e}")
+                ml_reason = "ML Err"
+
+        # --- Smart Order Book Analysis ---
+        ob_res = {'score': 0.5, 'imbalance': 0.5, 'veto': False, 'reason': 'No OB'}
+        if order_book_data:
+            ob_res = self._score_order_book(order_book_data, symbol=symbol)
 
-        if ob_data.get('veto', False):
+        # --- Final Hybrid Score ---
+        # If OB vetos (Spread too high OR Sell Wall), we force score down or WAIT
+        if ob_res.get('veto', False):
+            final_score = 0.0
             signal = 'WAIT'
-            reason_str = f"⛔ BLOCKED by OB Veto: {ob_data.get('reason')}"
-            
-        # ✅ استخدام العتبة المحقونة entry_threshold
-        elif final_score >= self.entry_threshold:
-            signal = 'BUY'
-            reason_str = f"✅ APPROVED: {final_score:.2f} >= {self.entry_threshold} | ML:{ml_score:.2f}"
-        
+            reason_str = f"⛔ {ob_res['reason']} | {ml_reason}"
         else:
-            signal = 'WAIT'
-            reason_str = f"❌ LOW SCORE: {final_score:.2f} < {self.entry_threshold} | ML:{ml_score:.2f}"
+            final_score = (ml_score * self.weight_ml) + (ob_res['score'] * self.weight_ob)
+            
+            if final_score >= self.entry_threshold:
+                signal = 'BUY'
+                reason_str = f"✅ GO: {final_score:.2f} | {ml_reason} | OB:{ob_res['score']:.2f}"
+            else:
+                signal = 'WAIT'
+                reason_str = f"📉 Low Score: {final_score:.2f} | {ml_reason}"
 
         return {
             'signal': signal,
             'confidence_prob': final_score,
             'ml_score': ml_score,
-            'ob_score': ob_data['score'],
-            'ob_data': ob_data,
-            'threshold': self.entry_threshold,
+            'ob_score': ob_res['score'],
+            'entry_price': float(order_book_data['asks'][0][0]) if order_book_data and order_book_data.get('asks') else 0.0,
             'reason': reason_str
         }