Update app.py

app.py

@@ -1,10 +1,9 @@
 # ==============================================================================
-# 🚀 app.py (V69.0 - GEM-Architect: Full Scale Production System)
+# 🚀 app.py (V70.0 - GEM-Architect: The 5-Layer Neural Core)
 # ==============================================================================
-# - Architecture: Titan V2 (ResNet) + Oracle V4.5 (LightGBM Exp.Ret).
-# - Removed: Legacy Pattern Engine columns and logic.
-# - Integrates: R2, Whale Monitor, News, Sentiment, Adaptive Hub, Tuner.
-# - UI: Full Gradio Dashboard with Charting, Logging, and Manual Controls.
+# - Architecture: PatternNet (ResNet) + Oracle V4.5 + Sniper V2.
+# - Logic: 5-Stage Precision Funnel (Breadth -> Neural -> External -> Micro -> Governance).
+# - UI: Full Gradio Dashboard with Real-time Monitoring.
 # ==============================================================================
 
 import os
@@ -109,7 +108,7 @@ class SystemState:
 
     def set_ready(self): 
         self.ready = True
-        self.last_cycle_logs = "✅ System Ready. Titan V2 (Pure ResNet) Online."
+        self.last_cycle_logs = "✅ System Ready. PatternNet (ResNet) Online."
         logger.info("✅ System State set to READY.")
     
     def set_cycle_start(self): 
@@ -207,7 +206,7 @@ async def auto_pilot_loop():
 async def lifespan(app: FastAPI):
     global r2, data_manager, ml_processor, adaptive_hub, trade_manager, whale_monitor, news_fetcher, senti_analyzer, sys_state
     
-    logger.info("\n🚀 [System] Startup Sequence (Titan V69.0 - Production)...")
+    logger.info("\n🚀 [System] Startup Sequence (Titan V70.0 - 5-Layer Core)...")
     try:
         # 1. Initialize R2 (Persistence)
         r2 = R2Service()
@@ -231,7 +230,7 @@ async def lifespan(app: FastAPI):
         adaptive_hub = AdaptiveHub(r2_service=r2)
         await adaptive_hub.initialize()
 
-        # 6. Initialize ML Processor (Titan V2 + Oracle)
+        # 6. Initialize ML Processor (PatternNet + Oracle + Sniper)
         ml_processor = MLProcessor(data_manager=data_manager)
         await ml_processor.initialize()
         
@@ -268,16 +267,16 @@ async def lifespan(app: FastAPI):
         logger.info("✅ [System] Shutdown Complete.")
 
 # ------------------------------------------------------------------------------
-# 7. Analysis Tasks (Layer 2)
+# 7. Analysis Tasks (Data Fetcher for Layers)
 # ------------------------------------------------------------------------------
-async def _analyze_symbol_task(candidate_data: Dict[str, Any]) -> Dict[str, Any]:
+async def _fetch_l2_data_task(candidate_data: Dict[str, Any]) -> Dict[str, Any]:
     """
-    Performs deep L2 analysis on a single symbol using MLProcessor.
-    Fetches required 5m, 15m, 1h, 4h data.
+    Fetches required 15m, 1h, 4h data for L2 (Pattern/Oracle/MC).
+    Does NOT run the models, just prepares the data package.
     """
     try:
         symbol = candidate_data['symbol']
-        required_tfs = ["5m", "15m", "1h", "4h"]
+        required_tfs = ["15m", "1h", "4h"] # PatternNet needs 15m, MC needs 1h
         
         # Concurrent Data Fetching
         data_tasks = [data_manager.get_latest_ohlcv(symbol, tf, limit=300) for tf in required_tfs]
@@ -288,7 +287,7 @@ async def _analyze_symbol_task(candidate_data: Dict[str, Any]) -> Dict[str, Any]
             if data and len(data) > 0: ohlcv_data[tf] = data
             
         # Validation: Ensure critical timeframes exist
-        if '1h' not in ohlcv_data or '5m' not in ohlcv_data: 
+        if '15m' not in ohlcv_data: 
             return None
         
         current_price = await data_manager.get_latest_price_async(symbol)
@@ -305,24 +304,22 @@ async def _analyze_symbol_task(candidate_data: Dict[str, Any]) -> Dict[str, Any]
             'strategy_type': candidate_data.get('strategy_type', 'NORMAL'), 
             'l1_score': candidate_data.get('l1_sort_score', 0)
         }
-        
-        # Call Processor (Titan V2 -> Oracle)
-        res = await ml_processor.process_compound_signal(raw_data)
-        if not res: return None
-        
-        # Pass through strategy type
-        res['strategy_type'] = candidate_data.get('strategy_type', 'NORMAL')
-        return res
+        return raw_data
+
     except Exception: 
         return None
 
 # ------------------------------------------------------------------------------
-# 8. Unified Logic Cycle (The Core Loop)
+# 8. Unified Logic Cycle (The 5-Layer Pipeline)
 # ------------------------------------------------------------------------------
 async def run_unified_cycle():
     """
-    Executes the full L1 -> L2 -> L3 -> L4 Analysis Pipeline.
-    Updated for Titan V2 (No Patterns logic).
+    💎 GEM-Architect: The 5-Layer Precision Pipeline
+    Layer 1: Breadth Filter (Data Manager)
+    Layer 2: Pattern + Oracle + MC (Score & Gate) -> Limit to Top 10
+    Layer 3: External Data (News + Whales) -> Re-Rank Top 10
+    Layer 4: Sniper (Micro-Structure) -> Pick Best 1
+    Layer 5: Governance (Senate) -> Final Approval & Execution
     """
     log_buffer = StringIO()
     def log_and_print(message):
@@ -347,160 +344,158 @@ async def run_unified_cycle():
                 pnl = ((curr_p - entry_p) / entry_p) * 100 if entry_p > 0 else 0
                 log_and_print(f"   🔒 {sym}: {pnl:+.2f}%")
 
-        # --- PHASE 1: L1 Screening ---
-        log_and_print(f"   [1/5] 🔍 L1 Screening (Bottom/Momentum)...")
-        candidates = await data_manager.layer1_rapid_screening(adaptive_hub_ref=adaptive_hub) 
-        if not candidates:
-            log_and_print("⚠️ No valid candidates found (Quality Filter).")
+        # ==============================================================================
+        # 🌊 LAYER 1: Market Breadth & Liquidity Filter
+        # ==============================================================================
+        log_and_print(f"   [1/5] 🌊 Layer 1: Market Breadth & Liquidity Screen...")
+        l1_candidates = await data_manager.layer1_rapid_screening(limit=200, adaptive_hub_ref=adaptive_hub) 
+        
+        if not l1_candidates:
+            log_and_print("⚠️ [Layer 1] No candidates found (Market unsafe or flat).")
             sys_state.set_cycle_end(logs=log_buffer.getvalue())
             return
             
-        # --- PHASE 2: L2 Deep Analysis ---
-        log_and_print(f"   [2/5] 🧠 L2 Deep Analysis ({len(candidates)} items)...")
-        tasks = [_analyze_symbol_task(c) for c in candidates]
-        results = await asyncio.gather(*tasks)
-        
-        # Filter Results
-        valid_l2 = [res for res in results if res is not None and res.get('is_valid', False)]
-        rejected_l2 = [res for res in results if res is not None and not res.get('is_valid', False)]
-        
-        rejected_l2.sort(key=lambda x: x.get('enhanced_final_score', 0.0), reverse=True)
-
-        if not valid_l2:
-            log_and_print("⚠️ No valid L2 candidates (Hard Gates Active). Top Rejected:")
-            # Updated Rejection Header (Removed Patterns)
-            rej_header = f"{'SYM':<9} | {'TITAN':<5} | {'MC':<5} | {'FINAL':<5} | {'REASON'}"
-            log_and_print(rej_header)
-            log_and_print("-" * 100)
-            
-            for rej in rejected_l2[:10]:
-                sym = rej['symbol']
-                tit = rej.get('titan_score', 0.0)
-                mc = rej.get('mc_score', 0.0)
-                fin = rej.get('enhanced_final_score', 0.0)
-                reason = rej.get('rejection_reason', 'Unknown')
-                log_and_print(f"{sym:<9} | {tit:.1f}  | {mc:.1f}  | {fin:.1f}  | {reason}")
-
+        # ==============================================================================
+        # 🧠 LAYER 2: Neural Analysis (PatternNet + Oracle + MC)
+        # ==============================================================================
+        log_and_print(f"   [2/5] 🧠 Layer 2: Neural Pattern Analysis ({len(l1_candidates)} coins)...")
+        
+        # Helper wrapper for concurrency
+        async def process_l2(cand):
+            raw_data = await _fetch_l2_data_task(cand)
+            if not raw_data: return None
+            return await ml_processor.execute_layer2_analysis(raw_data)
+
+        # Batch execute L2
+        l2_tasks = [process_l2(c) for c in l1_candidates]
+        l2_results = await asyncio.gather(*l2_tasks)
+        
+        # Filter failures (None)
+        valid_l2 = [res for res in l2_results if res is not None]
+        
+        # Sort by Composite Score and Keep TOP 10
+        valid_l2.sort(key=lambda x: x['l2_score'], reverse=True)
+        top_10_candidates = valid_l2[:10]
+
+        if not top_10_candidates:
+            log_and_print("⚠️ [Layer 2] All candidates failed Pattern Gate (<40%).")
+            # Log top rejected for debug
             sys_state.set_cycle_end(logs=log_buffer.getvalue())
             return
 
-        # Select Top Semi-Finalists
-        semi_finalists = sorted(valid_l2, key=lambda x: x.get('enhanced_final_score', 0.0), reverse=True)[:10]
-
-        # --- PHASE 3: L3 Deep Dive (Context & Oracle) ---
-        log_and_print(f"   [3/5] 📡 L3 Deep Dive (Whales/News) & Oracle Check...")
-        final_candidates = []
-
-        for sig in semi_finalists:
-            symbol = sig['symbol']
-            l2_score = sig.get('enhanced_final_score', 0.0)
+        # ==============================================================================
+        # 📡 LAYER 3: External Intelligence (News + Whales)
+        # ==============================================================================
+        log_and_print(f"   [3/5] 📡 Layer 3: External Intelligence Injection (Top {len(top_10_candidates)})...")
+        final_l3_list = []
+        
+        # Header for L3 Table
+        header = (f"{'SYM':<9} | {'PATT':<5} | {'MC':<4} | {'ORCL':<5} | "
+                  f"{'WHALE':<5} | {'NEWS':<5} | {'TOTAL':<6}")
+        log_and_print("-" * 60)
+        log_and_print(header)
+        
+        for cand in top_10_candidates:
+            symbol = cand['symbol']
+            l2_score = cand['l2_score']
             
             # Whale Analysis
-            whale_points = 0.0
-            try:
-                if whale_monitor:
-                    w_data = await whale_monitor.get_symbol_whale_activity(symbol, known_price=sig.get('current_price', 0))
-                    if w_data and w_data.get('data_available', False) and 'trading_signal' in w_data:
-                        signal = w_data['trading_signal']
-                        action = signal.get('action', 'HOLD')
-                        confidence = float(signal.get('confidence', 0.5))
-                        dynamic_impact = SystemLimits.L3_WHALE_IMPACT_MAX * confidence 
-                        if action == 'BUY': whale_points = dynamic_impact 
-                        elif action == 'SELL': whale_points = -dynamic_impact
-            except Exception: pass
+            whale_bonus = 0.0
+            if whale_monitor:
+                try:
+                    w_data = await whale_monitor.get_symbol_whale_activity(symbol, known_price=cand.get('current_price', 0))
+                    if w_data and w_data.get('trading_signal', {}).get('action') == 'BUY':
+                        conf = float(w_data.get('trading_signal', {}).get('confidence', 0.5))
+                        whale_bonus = SystemLimits.L3_WHALE_IMPACT_MAX * conf
+                except: pass
 
             # News Analysis
-            news_points = 0.0
-            try:
-                if news_fetcher and senti_analyzer:
+            news_bonus = 0.0
+            if news_fetcher and senti_analyzer:
+                try:
                     n_data = await news_fetcher.get_news(symbol)
-                    summary_text = n_data.get('summary', '')
-                    if "No specific news" not in summary_text:
-                        sent = senti_analyzer.polarity_scores(summary_text)
-                        compound_score = sent['compound']
-                        news_points = compound_score * SystemLimits.L3_NEWS_IMPACT_MAX
-            except Exception: pass
-
-            # Advanced MC (Placeholder for future expansion)
-            mc_a_points = 0.0
-            try:
-                raw_mc_a = await ml_processor.run_advanced_monte_carlo(symbol, '1h')
-                mc_a_points = max(-SystemLimits.L3_MC_ADVANCED_MAX, min(SystemLimits.L3_MC_ADVANCED_MAX, raw_mc_a))
-            except Exception: pass
-
-            # Calculate Final Score (Pre-Oracle)
-            final_score = l2_score + whale_points + news_points + mc_a_points
+                    summary = n_data.get('summary', '')
+                    if "No specific news" not in summary:
+                        sent = senti_analyzer.polarity_scores(summary)
+                        news_bonus = sent['compound'] * SystemLimits.L3_NEWS_IMPACT_MAX
+                except: pass
+
+            # Final Score Calculation
+            final_score = l2_score + whale_bonus + news_bonus
             
-            sig['whale_score'] = whale_points
-            sig['news_score'] = news_points
-            sig['mc_advanced_score'] = mc_a_points
-            sig['final_total_score'] = final_score 
-            final_candidates.append(sig)
+            cand['whale_score'] = whale_bonus
+            cand['news_score'] = news_bonus
+            cand['final_total_score'] = final_score 
+            
+            log_and_print(
+                f"{symbol:<9} | "
+                f"{cand['pattern_score']*100:.1f}  | "
+                f"{cand['mc_score']*100:.1f} | "
+                f"{cand['oracle_score']*100:.1f}  | "
+                f"{whale_bonus:+.1f}  | " 
+                f"{news_bonus:+.1f}  | "
+                f"{final_score:.1f}"
+            )
+            final_l3_list.append(cand)
 
-        final_candidates.sort(key=lambda x: x['final_total_score'], reverse=True)
+        # Re-sort based on L3 Total
+        final_l3_list.sort(key=lambda x: x['final_total_score'], reverse=True)
 
-        approved_signals = []
+        # ==============================================================================
+        # 🎯 LAYER 4: Sniper (Micro-Structure) - Find The One
+        # ==============================================================================
+        log_and_print(f"   [4/5] 🎯 Layer 4: Sniper Micro-Analysis (Scanning Top 5)...")
         
-        # ✅ UPDATED HEADER: Showing Oracle PnL (Exp.Ret) instead of Patterns
-        header = (f"{'SYM':<9} | {'TYPE':<8} | {'TITAN':<5} | {'MC':<4} | "
-                  f"{'NEWS':<5} | {'WHALE':<6} | {'FINAL':<6} | {'EXP.RET':<8} | {'STATUS'}")
-        
-        log_and_print("-" * 140)
-        log_and_print(header)
-        log_and_print("-" * 140)
+        best_candidate = None
+        best_sniper_score = -1.0
         
-        for sig in final_candidates:
-            symbol = sig['symbol']
+        # Scan top 5 only to save API/Processing
+        for cand in final_l3_list[:5]:
+            symbol = cand['symbol']
             
-            # Consult Updated Oracle
-            decision = await ml_processor.consult_oracle(sig)
+            # Fetch 1m Data & Order Book
+            t1m = await data_manager.get_latest_ohlcv(symbol, '1m', 500)
+            ob = await data_manager.get_order_book_snapshot(symbol)
             
-            action = decision.get('action', 'WAIT')
-            oracle_score = decision.get('oracle_score', 0.0) # Expected PnL from LightGBM
-            target_class = decision.get('target_class', '')
+            if not t1m or not ob: continue
             
-            status_str = "WAIT 🔴"
-            # If Oracle says Watch or Buy, we approve for Governance
-            if action == 'WATCH' or action == 'BUY':
-                status_str = f"✅ {target_class}"
-                sig.update(decision)
-                approved_signals.append(sig)
-
-            titan_d = sig.get('titan_score', 0.0)
-            news_d = sig.get('news_score', 0.0)
-            whale_d = sig.get('whale_score', 0.0)
-            mc_d = sig.get('mc_score', 0.0) 
-            final_d = sig.get('final_total_score', 0.0)
-            strat_type = sig.get('strategy_type', 'N/A')
-
-            log_and_print(
-                f"{symbol:<9} | "
-                f"{strat_type:<8} | "
-                f"{titan_d:.1f}  | "
-                f"{mc_d:.1f}  | "
-                f"{news_d:+.1f} | "  
-                f"{whale_d:+.1f}  | " 
-                f"{final_d:.1f}   | " 
-                f"{oracle_score*100:+.2f}%   | " # Display Expected Return
-                f"{status_str}"
-            )
-
-        # --- PHASE 4: Execution & Governance ---
-        if approved_signals:
-            log_and_print("-" * 140)
-            log_and_print(f"   [4/5] 🎯 L4 Sniper -> 🏛️ Governance -> 💰 Portfolio ({len(approved_signals)} candidates)...")
-            tm_log_buffer = StringIO()
+            # Run Sniper
+            sniper_res = await ml_processor.execute_layer4_sniper(symbol, t1m, ob)
             
-            # Capture Trade Manager Output
-            with redirect_stdout(tm_log_buffer), redirect_stderr(tm_log_buffer):
-                await trade_manager.select_and_execute_best_signal(approved_signals)
+            s_sig = sniper_res.get('signal', 'WAIT')
+            s_conf = sniper_res.get('confidence_prob', 0.0)
+            s_reason = sniper_res.get('reason', 'N/A')
             
-            tm_logs = tm_log_buffer.getvalue()
-            for line in tm_logs.splitlines():
-                if line.strip(): log_and_print(line.strip())
-        else:
-            log_and_print("   -> 🛑 No candidates passed Oracle's Golden Threshold.")
+            log_and_print(f"      🔭 {symbol:<6} -> Signal: {s_sig} | Conf: {s_conf:.2f} | {s_reason}")
+            
+            # Logic: Must be BUY and have highest score
+            if s_sig == 'BUY' and s_conf > best_sniper_score:
+                best_sniper_score = s_conf
+                cand['sniper_entry_price'] = sniper_res.get('entry_price', 0)
+                cand['sniper_score'] = s_conf
+                best_candidate = cand
+
+        if not best_candidate:
+            log_and_print("🛑 Layer 4: Sniper found no valid entry points in top candidates.")
+            sys_state.set_cycle_end(logs=log_buffer.getvalue())
+            return
 
+        # ==============================================================================
+        # 🏛️ LAYER 5: Governance & Execution
+        # ==============================================================================
+        log_and_print(f"   [5/5] 🏛️ Layer 5: Sending {best_candidate['symbol']} to Governance...")
+        
+        tm_log_buffer = StringIO()
+        
+        # Capture Trade Manager Output
+        with redirect_stdout(tm_log_buffer), redirect_stderr(tm_log_buffer):
+            # Pass as a list because TM expects a list of candidates to confirm
+            await trade_manager.select_and_execute_best_signal([best_candidate])
+        
+        tm_logs = tm_log_buffer.getvalue()
+        for line in tm_logs.splitlines():
+            if line.strip(): log_and_print(line.strip())
+            
         gc.collect()
         sys_state.set_cycle_end(logs=log_buffer.getvalue())
 
@@ -633,7 +628,7 @@ async def check_live_pnl_and_status(selected_view="Dual-Core (Hybrid)"):
             decision_data = trade.get('decision_data', {})
             gov_grade = decision_data.get('governance_grade', 'N/A')
             gov_score = decision_data.get('governance_score', 0.0)
-            oracle_score = decision_data.get('oracle_score', 0.0) # New Metric
+            oracle_score = decision_data.get('oracle_score', 0.0) 
             
             grade_color = "#ccc"
             if gov_grade == "ULTRA": grade_color = "#ff00ff" 
@@ -756,11 +751,17 @@ async def check_live_pnl_and_status(selected_view="Dual-Core (Hybrid)"):
         # --- DataFrames ---
         diag_data = await r2.get_diagnostic_stats_async()
         diag_list = []
-        # Removed Patterns
-        ordered_models = ["Titan", "Oracle", "Sniper", "MonteCarlo_L", "MonteCarlo_A", "News", "Governance"]
+        # Updated Keys: Titan -> Pattern (Backwards compatibility logic)
+        ordered_models = ["Titan", "Pattern", "Oracle", "Sniper", "MonteCarlo_L", "MonteCarlo_A", "News", "Governance"]
         
         for m in ordered_models:
             stats = diag_data.get(m, {"wins": 0, "losses": 0, "pnl": 0.0, "profit_accum": 0.0, "loss_accum": 0.0})
+            
+            # Merge Titan & Pattern stats if needed
+            if m == "Pattern" and "Titan" in diag_data:
+                # Optional: You can choose to display them separate or merged
+                pass 
+                
             profit_accum = stats.get('profit_accum', 0.0)
             loss_accum = stats.get('loss_accum', 0.0)
             if profit_accum == 0.0 and loss_accum == 0.0 and stats['pnl'] != 0.0:
@@ -768,7 +769,11 @@ async def check_live_pnl_and_status(selected_view="Dual-Core (Hybrid)"):
                  else: loss_accum = abs(stats['pnl'])
 
             pnl_str = format_pnl_split(profit_accum, loss_accum)
-            diag_list.append([m, stats['wins'], stats['losses'], pnl_str])
+            # Rename Titan to Pattern in UI if found
+            display_name = "Pattern (Net)" if m == "Titan" else m
+            
+            if stats['wins'] + stats['losses'] > 0:
+                diag_list.append([display_name, stats['wins'], stats['losses'], pnl_str])
         
         diag_df = pd.DataFrame(diag_list, columns=["Model", "Wins", "Losses", "PnL (USD)"])
 
@@ -835,9 +840,9 @@ async def check_live_pnl_and_status(selected_view="Dual-Core (Hybrid)"):
 def create_gradio_ui():
     custom_css = ".gradio-container {background:#0b0f19} .dataframe {background:#1a1a1a!important} .html-box {min-height:180px}"
     
-    with gr.Blocks(title="Titan V69.0 (Integrated)", css=custom_css) as demo:
+    with gr.Blocks(title="Titan V70.0 (5-Layer Core)", css=custom_css) as demo:
         
-        gr.Markdown("# 🚀 Titan V69.0 (Pure Titan + Oracle V4.5)")
+        gr.Markdown("# 🚀 Titan V70.0 (Neural Pattern Core + 5-Layer Funnel)")
         
         with gr.Row():
             # LEFT: Chart & Controls