Update app.py

app.py

@@ -7,7 +7,6 @@ import torch.nn.functional as F
 import requests
 import os
 import time
-import threading
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 from sklearn.preprocessing import StandardScaler
@@ -16,7 +15,6 @@ from sklearn.preprocessing import StandardScaler
 API_KEY = os.getenv("TWELVEDATA_KEY")
 NTFY_TOPIC = os.getenv("NTFY_TOPIC")
 
-# The Constellation Basket
 TARGET_PAIR = "EUR/USD"
 SYMBOLS = ["EUR/USD", "GBP/USD", "USD/JPY", "XAU/USD"] 
 TIMEFRAME = "15min"
@@ -24,13 +22,14 @@ LOOKBACK = 30
 
 # Global State
 GLOBAL_STATE = {
-    "base_model": None,   # The Transformer (The Teacher)
-    "shadow_model": None, # The Online Learner (The Student)
+    "base_model": None,   # The Transformer (Teacher)
+    "shadow_model": None, # The Online Learner (Student)
     "last_trade": None,
-    "last_run_time": 0
+    "scaler": None,
+    "is_base_trained": False
 }
 
-# --- 2. THE TEACHER: CONSTELLATION TRANSFORMER ---
+# --- 2. THE TEACHER: CONSTELLATION TRANSFORMER (Your Original Model) ---
 class PositionalEncoding(nn.Module):
     def __init__(self, d_model, max_len=5000):
         super(PositionalEncoding, self).__init__()
@@ -62,42 +61,40 @@ class ConstellationTransformer(nn.Module):
         mu = self.z_mu(context)
         return pi, sigma, mu
 
-# --- 3. THE STUDENT: META SHADOW LEARNER (ONLINE) ---
-class MetaShadowLearner(nn.Module):
-    def __init__(self, input_size=4, hidden_size=32):
-        super(MetaShadowLearner, self).__init__()
-        # Simple MLP that learns FAST
+def mdn_loss(pi, sigma, mu, y):
+    if y.dim() == 1: y = y.unsqueeze(1)
+    dist = torch.distributions.Normal(loc=mu, scale=sigma)
+    log_prob = dist.log_prob(y)
+    loss = -torch.logsumexp(torch.log(pi + 1e-8) + log_prob, dim=1)
+    return torch.mean(loss)
+
+# --- 3. THE STUDENT: ONLINE META LEARNER (New) ---
+class OnlineMetaLearner(nn.Module):
+    def __init__(self, input_dim=3, hidden_dim=32):
+        super(OnlineMetaLearner, self).__init__()
+        # It takes [Base_Pred, Base_Sigma, Volatility] as input
         self.net = nn.Sequential(
-            nn.Linear(input_size, hidden_size),
-            nn.Tanh(), # Tanh for non-linearity
-            nn.Linear(hidden_size, hidden_size),
+            nn.Linear(input_dim, hidden_dim),
+            nn.Tanh(), # Tanh is safer for corrections (bound between -1 and 1)
+            nn.Linear(hidden_dim, hidden_dim),
             nn.ReLU(),
-            nn.Linear(hidden_size, 1) # Output: The Correction Factor
+            nn.Linear(hidden_dim, 1) # Outputs the CORRECTION
         )
-        # High learning rate because we learn sample-by-sample
+        # Fast learning rate for "On-the-Spot" adaptation
         self.optimizer = torch.optim.Adam(self.parameters(), lr=0.01)
         self.loss_fn = nn.MSELoss()
 
     def forward(self, x):
         return self.net(x)
 
-    def learn_on_the_spot(self, inputs, target_error):
-        """
-        Takes one sample, calculates loss, updates weights instantly.
-        No batches. Pure online learning.
-        """
+    def learn_step(self, x, target_error):
         self.train()
         self.optimizer.zero_grad()
-        
-        predicted_error = self.forward(inputs)
-        
-        # We want the shadow model to predict the error of the base model
-        loss = self.loss_fn(predicted_error, target_error)
-        
+        pred_correction = self.forward(x)
+        loss = self.loss_fn(pred_correction, target_error)
         loss.backward()
         self.optimizer.step()
-        
-        return predicted_error.item()
+        return pred_correction.item()
 
 # --- 4. DATA PIPELINE ---
 def get_constellation_data():
@@ -114,244 +111,256 @@ def get_constellation_data():
             df = df[['close']].astype(float)
             df.rename(columns={'close': sym}, inplace=True)
             dfs.append(df)
-            time.sleep(0.1) 
+            time.sleep(0.1)
         except: pass
     
     if not dfs: return None, "❌ Failed to fetch data."
     master_df = pd.concat(dfs, axis=1).ffill().dropna()
     return master_df, "✅ Constellation Aligned"
 
-def prepare_tensors(master_df):
-    # Returns for ALL Pairs
+def get_events_data():
+    try:
+        url = "https://nfs.faireconomy.media/ff_calendar_thisweek.json"
+        r = requests.get(url, headers={"User-Agent": "V23/1.0"}, timeout=5)
+        data = r.json()
+        parsed = []
+        impact_map = {'Low': 1, 'Medium': 2, 'High': 3}
+        for i in data:
+            if i.get('country') in ['EUR', 'USD']:
+                dt = pd.to_datetime(i.get('date'), utc=True).tz_localize(None)
+                imp = impact_map.get(i.get('impact'), 0)
+                parsed.append({'DateTime': dt, 'Impact_Score': imp})
+        df = pd.DataFrame(parsed)
+        if not df.empty: df = df.sort_values('DateTime').set_index('DateTime')
+        return df
+    except: return pd.DataFrame()
+
+def prepare_tensors(master_df, event_df):
+    if not event_df.empty:
+        merged = pd.merge_asof(master_df, event_df, left_index=True, right_index=True, direction='backward', tolerance=pd.Timedelta('4 hours')).fillna(0)
+    else:
+        merged = master_df.copy(); merged['Impact_Score'] = 0
+    merged['Surprise'] = 0.0
+
     feature_cols = []
     for sym in SYMBOLS:
         col_name = f"{sym}_ret"
-        master_df[col_name] = master_df[sym].pct_change().fillna(0)
+        merged[col_name] = merged[sym].pct_change().fillna(0)
         feature_cols.append(col_name)
+    feature_cols.extend(['Surprise', 'Impact_Score'])
     
-    # Scale
     scaler = StandardScaler()
-    data_scaled = scaler.fit_transform(master_df[feature_cols].values)
+    data_scaled = scaler.fit_transform(merged[feature_cols].values)
     
-    # Windows
     X_data = []
     for i in range(LOOKBACK, len(data_scaled)):
         X_data.append(data_scaled[i-LOOKBACK:i])
     
     X_tensor = torch.FloatTensor(np.array(X_data))
     
-    # Metadata
     target_idx = 0 
     ret_mean = scaler.mean_[target_idx]
     ret_scale = scaler.scale_[target_idx]
-    ref_prices = master_df[TARGET_PAIR].values[LOOKBACK:]
+    ref_prices = merged[TARGET_PAIR].values[LOOKBACK:]
     
-    return X_tensor, master_df.index[LOOKBACK:], ref_prices, ret_mean, ret_scale, data_scaled
+    return X_tensor, merged.index[LOOKBACK:], ref_prices, ret_mean, ret_scale, data_scaled
 
 # --- 5. CORE LOGIC ---
 def send_ntfy(message):
     if not NTFY_TOPIC: return
     try:
-        requests.post(f"https://ntfy.sh/{NTFY_TOPIC}", data=message.encode('utf-8'), headers={"Title": "Shadow FX V5", "Priority": "high"})
+        requests.post(f"https://ntfy.sh/{NTFY_TOPIC}", data=message.encode('utf-8'), headers={"Title": "Hybrid V4", "Priority": "high"})
     except: pass
 
+def hard_reset():
+    GLOBAL_STATE["base_model"] = None
+    GLOBAL_STATE["shadow_model"] = None
+    GLOBAL_STATE["is_base_trained"] = False
+    return None, "<div>♻️ Memory Wiped.</div>", "Reset."
+
 def run_analysis():
     log_buffer = []
     
-    # 1. Initialize Models
+    # 1. Initialize Base Model (Teacher)
     if GLOBAL_STATE["base_model"] is None:
-        GLOBAL_STATE["base_model"] = ConstellationTransformer(input_dim=4, d_model=64)
+        GLOBAL_STATE["base_model"] = ConstellationTransformer(input_dim=6, d_model=64, num_layers=2)
         log_buffer.append("🧠 Base Transformer Initialized")
-
-    # Always re-init shadow model for fresh "session" learning or keep it if you want long term
-    GLOBAL_STATE["shadow_model"] = MetaShadowLearner(input_size=4) 
     
+    # 2. Initialize Shadow Model (Student) - Always fresh or persistent?
+    # Let's keep it persistent so it gets smarter over time, but reset if hard_reset called
+    if GLOBAL_STATE["shadow_model"] is None:
+        GLOBAL_STATE["shadow_model"] = OnlineMetaLearner(input_dim=3)
+        log_buffer.append("👻 Shadow Learner Initialized")
+        
     base_model = GLOBAL_STATE["base_model"]
     shadow_model = GLOBAL_STATE["shadow_model"]
     
-    # 2. Get Data
+    # 3. Data
     master_df, msg = get_constellation_data()
     if master_df is None: return None, msg, msg
+    event_df = get_events_data()
+    X_tensor, dates, ref_prices, ret_mean, ret_std, raw_features = prepare_tensors(master_df, event_df)
     
-    X_tensor, dates, ref_prices, ret_mean, ret_std, raw_scaled_features = prepare_tensors(master_df)
+    # 4. Train Base Model (The "Pre-Knowledge")
+    # We KEEP this to prevent the "Drunk" zig-zags. The Base Model must be smart first.
+    if not GLOBAL_STATE["is_base_trained"]:
+        log_buffer.append("⚙️ Training Base Transformer (50 Epochs)...")
+        optimizer = torch.optim.Adam(base_model.parameters(), lr=0.005)
+        base_model.train()
+        
+        train_X = X_tensor[:-1]
+        actual_returns = np.diff(ref_prices) / ref_prices[:-1]
+        actual_returns_scaled = (actual_returns - ret_mean) / ret_std
+        train_y = torch.FloatTensor(actual_returns_scaled).unsqueeze(1)
+        train_X = train_X[:len(train_y)]
+
+        dataset = torch.utils.data.TensorDataset(train_X, train_y)
+        loader = torch.utils.data.DataLoader(dataset, batch_size=32, shuffle=True)
+        
+        for epoch in range(50):
+            for batch_X, batch_y in loader:
+                optimizer.zero_grad()
+                pi, sigma, mu = base_model(batch_X)
+                loss = mdn_loss(pi, sigma, mu, batch_y)
+                loss.backward()
+                optimizer.step()
+                
+        GLOBAL_STATE["is_base_trained"] = True
+        log_buffer.append("✅ Base Calibration Complete.")
+
+    # 5. HYBRID INFERENCE LOOP
+    # We now run through the data again.
+    # Base Model predicts -> Shadow Model corrects -> Weights update -> Next candle
+    
+    log_buffer.append("🧬 Running Online Adaptation Loop...")
     
-    # 3. THE LIVE LEARNING LOOP
     final_preds = []
     base_preds = []
-    shadow_corrections = []
-    
-    log_buffer.append("🧬 Shadow Model learning on the spot...")
+    corrections = []
     
     base_model.eval()
     
-    # We iterate up to len-1 because we need the 'next' value to calculate error
-    loop_limit = len(X_tensor) - 1
+    # We loop through history to let the Shadow Model "learn" the Base Model's weaknesses
+    loop_len = len(X_tensor) - 1
     
-    for i in range(loop_limit):
+    for i in range(loop_len):
         
-        # A. Base Model Prediction
+        # A. Base Model Prediction (Frozen weights here)
         with torch.no_grad():
-            curr_input = X_tensor[i].unsqueeze(0)
-            pi, sigma, mu = base_model(curr_input)
+            inp = X_tensor[i].unsqueeze(0)
+            pi, sigma, mu = base_model(inp)
             max_idx = torch.argmax(pi, dim=1)
-            base_pred_ret = mu[0, max_idx].item()
+            base_ret = mu[0, max_idx].item()
             base_sigma = sigma[0, max_idx].item()
         
-        # B. Construct Shadow Inputs
-        last_features = raw_scaled_features[LOOKBACK + i - 1] 
-        shadow_input = torch.tensor([[base_pred_ret, base_sigma, last_features[0], last_features[3]]], dtype=torch.float32)
+        # B. Prepare Shadow Input
+        # [Base_Prediction, Base_Confidence, Volatility]
+        # Volatility is approx from raw features (feature 0 is EURUSD ret)
+        vol = abs(raw_features[LOOKBACK+i][0]) 
+        shadow_in = torch.tensor([[base_ret, base_sigma, vol]], dtype=torch.float32)
         
-        # C. Shadow Prediction
+        # C. Shadow Prediction (Correction)
+        # Note: We call forward(), not learn_step() yet because we don't know the future
         with torch.no_grad():
-            correction = shadow_model(shadow_input).item()
-        
-        final_ret_pred = base_pred_ret + correction
+            correction = shadow_model(shadow_in).item()
+            
+        final_ret = base_ret + correction
         
-        # D. The Truth Revealed (Next Candle)
-        actual_ret_raw = (ref_prices[i+1] - ref_prices[i]) / ref_prices[i]
-        actual_ret_scaled = (actual_ret_raw - ret_mean) / ret_std
+        # D. Get Real Next Value
+        real_ret_raw = (ref_prices[i+1] - ref_prices[i]) / ref_prices[i]
+        real_ret_scaled = (real_ret_raw - ret_mean) / ret_std
         
-        # E. Base Model Error
-        base_error = actual_ret_scaled - base_pred_ret
+        # E. Calculate Error for Shadow
+        # The target for the Shadow is: "What should I have added to Base to make it perfect?"
+        target_correction = real_ret_scaled - base_ret
+        target_tensor = torch.tensor([[target_correction]], dtype=torch.float32)
         
-        # F. TEACH THE SHADOW
-        target_tensor = torch.tensor([[base_error]], dtype=torch.float32)
-        shadow_model.learn_on_the_spot(shadow_input, target_tensor)
+        # F. LEARN ON THE SPOT
+        shadow_model.learn_step(shadow_in, target_tensor)
         
-        base_preds.append(base_pred_ret)
-        shadow_corrections.append(correction)
-        final_preds.append(final_ret_pred)
+        base_preds.append(base_ret)
+        corrections.append(correction)
+        final_preds.append(final_ret)
 
-    # --- FIXING THE ARRAY ALIGNMENT ---
-    # The loop runs 'loop_limit' times.
-    # If len(X) is 100, loop_limit is 99.
-    # We generated 99 predictions.
-    # Prediction i=0 corresponds to the move from Price[0] to Price[1].
-    # So we align with dates[1:] and ref_prices[1:].
-    
-    # Slice the data to match the prediction count
+    # 6. Reconstruction & Plotting
     plot_dates = dates[1:1+len(final_preds)]
     plot_actual = ref_prices[1:1+len(final_preds)]
     
-    # Reconstruction
-    pred_prices = []
-    curr_price = ref_prices[0] # Start from known point
+    # Reconstruct prices from returns
+    pred_prices_base = []
+    pred_prices_final = []
     
-    for k, ret in enumerate(final_preds):
-        real_ret = (ret * ret_std) + ret_mean
-        # To avoid compounding error drift in visualization, we use the ACTUAL previous price
-        # to calculate the NEXT predicted price.
-        prev_actual_price = ref_prices[k] 
-        next_price = prev_actual_price * (1 + real_ret)
-        pred_prices.append(next_price)
-
-    # Create DataFrame (Safe Mode)
-    try:
-        df = pd.DataFrame({
-            'Close': plot_actual,
-            'Final_Pred': pred_prices,
-            'Base_Pred': [ref_prices[k] * (1 + (bp * ret_std) + ret_mean) for k, bp in enumerate(base_preds)],
-            'Correction': shadow_corrections
-        }, index=plot_dates)
-    except ValueError as e:
-        return None, f"❌ Data Size Mismatch: {e}", "\n".join(log_buffer)
-
-    # Calculate Z-Score
-    df['Gap'] = df['Final_Pred'] - df['Close']
-    df['Z_Score'] = (df['Gap'] - df['Gap'].rolling(50).mean()) / (df['Gap'].rolling(50).std() + 1e-9)
-    
-    if len(df) < 2: return None, "Not enough data yet.", "Waiting..."
-
-    last_z = df['Z_Score'].iloc[-1]
-    last_price = df['Close'].iloc[-1]
-    
-    # Signals
-    status = "NEUTRAL"
-    color = "gray"
-    if last_z > 2.0:
-        status = "BUY SIGNAL"
-        color = "#00ff00"
-        if GLOBAL_STATE["last_trade"] != "BUY":
-            send_ntfy(f"🚀 BUY EURUSD | Smart-Z: {last_z:.2f}")
-            GLOBAL_STATE["last_trade"] = "BUY"
-    elif last_z < -2.0:
-        status = "SELL SIGNAL"
-        color = "#ff0000"
-        if GLOBAL_STATE["last_trade"] != "SELL":
-            send_ntfy(f"🔻 SELL EURUSD | Smart-Z: {last_z:.2f}")
-            GLOBAL_STATE["last_trade"] = "SELL"
-            
-    GLOBAL_STATE["last_run_time"] = time.time()
-
-    # --- PLOTTING ---
-    fig = make_subplots(rows=3, cols=1, shared_xaxes=True, 
-                        vertical_spacing=0.05, 
-                        row_heights=[0.5, 0.25, 0.25],
-                        subplot_titles=("Price vs Self-Learning Prediction", "Shadow Correction (The 'Boost')", "Smart Divergence"))
+    for k in range(len(final_preds)):
+        prev_p = ref_prices[k] # Use actual previous to prevent drift
+        
+        # Base
+        b_ret = (base_preds[k] * ret_std) + ret_mean
+        pred_prices_base.append(prev_p * (1 + b_ret))
+        
+        # Final
+        f_ret = (final_preds[k] * ret_std) + ret_mean
+        pred_prices_final.append(prev_p * (1 + f_ret))
 
-    # 1. Price
-    fig.add_trace(go.Scatter(x=df.index, y=df['Close'], mode='lines', name='Actual Price', line=dict(color='gray', width=1)), row=1, col=1)
-    fig.add_trace(go.Scatter(x=df.index, y=df['Base_Pred'], mode='lines', name='Base Model', line=dict(color='cyan', width=1, dash='dot')), row=1, col=1)
-    fig.add_trace(go.Scatter(x=df.index, y=df['Final_Pred'], mode='lines', name='Adapted Model', line=dict(color='#ffff00', width=2)), row=1, col=1)
+    df = pd.DataFrame({
+        'Close': plot_actual,
+        'Base': pred_prices_base,
+        'Final': pred_prices_final,
+        'Correction': corrections
+    }, index=plot_dates)
 
-    # 2. Correction
-    fig.add_trace(go.Bar(x=df.index, y=df['Correction'], name='AI Correction', marker_color='purple'), row=2, col=1)
+    # Z-Score
+    df['Gap'] = df['Final'] - df['Close']
+    df['Z'] = (df['Gap'] - df['Gap'].rolling(50).mean()) / (df['Gap'].rolling(50).std() + 1e-9)
     
-    # 3. Z-Score
-    fig.add_trace(go.Bar(x=df.index, y=df['Z_Score'], name='Smart Z-Score', marker_color=df['Z_Score'].apply(lambda x: 'green' if x>0 else 'red')), row=3, col=1)
-    fig.add_hline(y=2.0, line_dash="dot", line_color="green", row=3, col=1)
-    fig.add_hline(y=-2.0, line_dash="dot", line_color="red", row=3, col=1)
+    if len(df) > 0:
+        last_z = df['Z'].iloc[-1]
+        last_p = df['Close'].iloc[-1]
+        
+        status = "NEUTRAL"
+        color = "gray"
+        if last_z > 2.0: status, color = "BUY SIGNAL", "green"
+        if last_z < -2.0: status, color = "SELL SIGNAL", "red"
+        
+        # Check notification
+        if "SIGNAL" in status and GLOBAL_STATE["last_trade"] != status:
+            send_ntfy(f"{status} EURUSD | Z: {last_z:.2f}")
+            GLOBAL_STATE["last_trade"] = status
 
-    fig.update_layout(template="plotly_dark", height=900, title=f"V5 ShadowNet: {status}")
-    
-    info_html = f"""
-    <div style="text-align: center; padding: 10px; background-color: {color}; color: white; border-radius: 5px;">
-        <h2>{status}</h2>
-        <p>Smart-Z: {last_z:.3f}</p>
-        <small>The yellow line learns from the cyan line's mistakes in real-time.</small>
-    </div>
-    """
+        fig = make_subplots(rows=3, cols=1, shared_xaxes=True, row_heights=[0.5, 0.25, 0.25],
+                            subplot_titles=("Hybrid Price Model", "AI Correction (Shadow)", "Divergence"))
+        
+        # 1. Price
+        fig.add_trace(go.Scatter(x=df.index, y=df['Close'], name='Price', line=dict(color='gray')), row=1, col=1)
+        fig.add_trace(go.Scatter(x=df.index, y=df['Base'], name='Base Transformer', line=dict(color='cyan', dash='dot')), row=1, col=1)
+        fig.add_trace(go.Scatter(x=df.index, y=df['Final'], name='Adapted (Shadow)', line=dict(color='yellow', width=2)), row=1, col=1)
+        
+        # 2. Correction
+        fig.add_trace(go.Bar(x=df.index, y=df['Correction'], name='Learned Correction', marker_color='purple'), row=2, col=1)
+        
+        # 3. Z
+        fig.add_trace(go.Bar(x=df.index, y=df['Z'], name='Z-Score', marker_color=df['Z'].apply(lambda x: 'green' if x>0 else 'red')), row=3, col=1)
+        fig.add_hline(y=2, line_dash="dot", row=3, col=1); fig.add_hline(y=-2, line_dash="dot", row=3, col=1)
+        
+        fig.update_layout(template="plotly_dark", height=800, title=f"Hybrid V4: {status}")
+        
+        info = f"<div style='background:{color};color:white;padding:10px;text-align:center'><h3>{status}</h3>Z: {last_z:.3f}</div>"
+        return fig, info, "\n".join(log_buffer)
     
-    return fig, info_html, "\n".join(log_buffer)
-
-def hard_reset():
-    GLOBAL_STATE["base_model"] = None
-    GLOBAL_STATE["shadow_model"] = None
-    return None, "Memory Wiped", "Reset Complete"
+    return None, "No Data", "Wait"
 
-# --- 6. BACKGROUND WORKER (Auto-Loop) ---
-def background_loop():
-    while True:
-        try:
-            print("⏰ Background Scan Running...")
-            run_analysis()
-            print("✅ Scan Complete. Sleeping...")
-        except Exception as e:
-            print(f"❌ Background Error: {e}")
-        time.sleep(900) # Run every 15 minutes (900 seconds)
-
-# Start Background Thread
-t = threading.Thread(target=background_loop, daemon=True)
-t.start()
-
-# --- 7. UI ---
-with gr.Blocks(title="ShadowFX V5") as app:
-    gr.Markdown("# 🧬 V5 Meta-Shadow (Self-Adaptive)")
-    gr.Markdown("This model has **no fixed weights**. It watches the base model, calculates its error on every candle, and updates its own brain instantly to correct the next prediction.")
-    
+# --- 6. UI ---
+with gr.Blocks(title="Hybrid V4") as app:
+    gr.Markdown("# 👁️ Hybrid V4: Transformer + Shadow Learner")
     with gr.Row():
-        refresh_btn = gr.Button("⚡ Run Analysis Now", variant="primary")
-        reset_btn = gr.Button("🧠 Wipe Memory", variant="stop")
-    
-    status_box = gr.HTML()
-    plot = gr.Plot()
-    logs = gr.Textbox(label="System Logs")
-    
-    refresh_btn.click(fn=run_analysis, outputs=[plot, status_box, logs])
-    reset_btn.click(fn=hard_reset, outputs=[plot, status_box, logs])
+        r = gr.Button("🔄 Scan", variant="primary")
+        w = gr.Button("⚠️ Wipe", variant="stop")
+    s = gr.HTML()
+    p = gr.Plot()
+    l = gr.Textbox()
     
-    # Load on start
-    app.load(fn=run_analysis, outputs=[plot, status_box, logs])
+    r.click(run_analysis, outputs=[p, s, l])
+    w.click(hard_reset, outputs=[p, s, l])
+    app.load(run_analysis, outputs=[p, s, l])
 
 if __name__ == "__main__":
     app.launch(ssr_mode=False)