Update app.py

app.py

@@ -6,19 +6,17 @@ import math
 import aiohttp
 import pandas as pd
 import numpy as np
-import tensorflow as tf
 from aiohttp import web
-from tensorflow.keras import layers, models, callbacks
-from sklearn.preprocessing import StandardScaler
+from sklearn.ensemble import RandomForestRegressor
 from concurrent.futures import ThreadPoolExecutor
 
+# --- CONFIGURATION ---
 SYMBOL_KRAKEN = "BTC/USD"
 PORT = 7860
 BROADCAST_RATE = 1.0
 PREDICTION_HORIZON = 100
 MAX_HISTORY = 5000
-TRAIN_INTERVAL = 600
-LOOKBACK_WINDOW = 60
+TRAIN_INTERVAL = 300
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
 
@@ -26,7 +24,7 @@ market_state = {
     "ohlc_history": [],
     "ready": False,
     "model": None,
-    "scaler": None,
+    "model_residuals": None,
     "last_training_time": 0,
     "last_price": 0,
     "price_change": 0
@@ -36,7 +34,7 @@ connected_clients = set()
 executor = ThreadPoolExecutor(max_workers=1)
 
 def calculate_indicators(candles):
-    if len(candles) < LOOKBACK_WINDOW + PREDICTION_HORIZON:
+    if len(candles) < 100:
         return None
     
     df = pd.DataFrame(candles)
@@ -44,31 +42,37 @@ def calculate_indicators(candles):
     for c in cols:
         df[c] = df[c].astype(float)
 
+    # Moving Averages
     df['ema20'] = df['close'].ewm(span=20, adjust=False).mean()
     df['ema50'] = df['close'].ewm(span=50, adjust=False).mean()
     
+    # Bollinger Bands
     df['std'] = df['close'].rolling(window=20).std()
     df['bb_upper'] = df['ema20'] + (df['std'] * 2)
     df['bb_lower'] = df['ema20'] - (df['std'] * 2)
 
+    # RSI
     delta = df['close'].diff()
     gain = (delta.where(delta > 0, 0)).rolling(window=14).mean()
     loss = (-delta.where(delta < 0, 0)).rolling(window=14).mean()
     rs = gain / loss
     df['rsi'] = 100 - (100 / (1 + rs))
 
+    # MACD
     k = df['close'].ewm(span=12, adjust=False).mean()
     d = df['close'].ewm(span=26, adjust=False).mean()
     df['macd'] = k - d
     df['macd_signal'] = df['macd'].ewm(span=9, adjust=False).mean()
     df['macd_hist'] = df['macd'] - df['macd_signal']
 
+    # ATR
     df['tr0'] = abs(df['high'] - df['low'])
     df['tr1'] = abs(df['high'] - df['close'].shift())
     df['tr2'] = abs(df['low'] - df['close'].shift())
     df['tr'] = df[['tr0', 'tr1', 'tr2']].max(axis=1)
     df['atr'] = df['tr'].rolling(window=14).mean()
 
+    # Features
     df['dist_ema20'] = (df['close'] - df['ema20']) / df['ema20']
     df['dist_ema50'] = (df['close'] - df['ema50']) / df['ema50']
     df['bb_width'] = (df['bb_upper'] - df['bb_lower']) / df['ema20']
@@ -76,14 +80,22 @@ def calculate_indicators(candles):
     df['vol_change'] = df['volume'].pct_change()
     df['log_ret'] = np.log(df['close'] / df['close'].shift(1))
 
+    # Time encoding
     df['datetime'] = pd.to_datetime(df['time'], unit='s')
     df['hour_sin'] = np.sin(2 * np.pi * df['datetime'].dt.hour / 24)
     df['hour_cos'] = np.cos(2 * np.pi * df['datetime'].dt.hour / 24)
 
-    return df.dropna()
+    # Lag Features
+    for lag in [1, 2, 3, 5, 8]:
+        df[f'rsi_lag{lag}'] = df['rsi'].shift(lag)
+        df[f'macd_hist_lag{lag}'] = df['macd_hist'].shift(lag)
+        df[f'log_ret_lag{lag}'] = df['log_ret'].shift(lag)
+        df[f'vol_change_lag{lag}'] = df['vol_change'].shift(lag)
+
+    return df
 
 def train_model(df):
-    logging.info(f"Training CNN Model (Candle Output) on {len(df)} candles...")
+    logging.info(f"Training ML Model on {len(df)} candles...")
     
     feature_cols = [
         'rsi', 'macd_hist', 'atr', 
@@ -93,75 +105,55 @@ def train_model(df):
         'hour_sin', 'hour_cos'
     ]
     
-    data_features = df[feature_cols].values
-    scaler = StandardScaler()
-    data_scaled = scaler.fit_transform(data_features)
-    
-    close_prices = df['close'].values
-    high_prices = df['high'].values
-    low_prices = df['low'].values
+    for lag in [1, 2, 3, 5, 8]:
+        feature_cols.extend([
+            f'rsi_lag{lag}', f'macd_hist_lag{lag}', 
+            f'log_ret_lag{lag}', f'vol_change_lag{lag}'
+        ])
     
-    targets = []
-    
-    # Create targets: 2 values per step (Return, Range)
-    # Range is normalized by price to make it scale-invariant
-    for i in range(len(close_prices) - PREDICTION_HORIZON):
-        current_close = close_prices[i]
-        
-        step_targets = []
-        for h in range(1, PREDICTION_HORIZON + 1):
-            future_idx = i + h
-            # 1. Cumulative Return from current step to future step
-            ret = (close_prices[future_idx] - current_close) / current_close
-            
-            # 2. Volatility/Range of that future candle ((High - Low) / Close)
-            rng = (high_prices[future_idx] - low_prices[future_idx]) / close_prices[future_idx]
-            
-            step_targets.extend([ret, rng])
-            
-        targets.append(step_targets)
+    data = df.dropna().copy()
     
-    targets = np.array(targets)
+    # Prepare targets (Future Returns relative to Current Price)
+    target_cols_dict = {}
+    target_names = []
     
-    X = []
-    y = []
+    for i in range(1, PREDICTION_HORIZON + 1):
+        col_name = f'target_return_{i}'
+        # Return at step i relative to step 0
+        target_cols_dict[col_name] = (data['close'].shift(-i) - data['close']) / data['close']
+        target_names.append(col_name)
     
-    valid_length = len(targets) - LOOKBACK_WINDOW
-    if valid_length <= 0:
-        return None, None
+    targets_df = pd.DataFrame(target_cols_dict, index=data.index)
+    data = pd.concat([data, targets_df], axis=1).dropna()
 
-    for i in range(valid_length):
-        X.append(data_scaled[i : i + LOOKBACK_WINDOW])
-        y.append(targets[i + LOOKBACK_WINDOW - 1])
-        
-    X = np.array(X)
-    y = np.array(y)
-    
-    if len(X) < 100:
+    if len(data) < 200:
         return None, None
 
-    # Model architecture for multi-output regression
-    model = models.Sequential([
-        layers.Input(shape=(LOOKBACK_WINDOW, len(feature_cols))),
-        layers.Conv1D(filters=64, kernel_size=3, activation='relu', padding='same'),
-        layers.MaxPooling1D(pool_size=2),
-        layers.Dropout(0.2),
-        layers.Conv1D(filters=32, kernel_size=3, activation='relu', padding='same'),
-        layers.GlobalAveragePooling1D(),
-        layers.Dense(64, activation='relu'),
-        layers.Dropout(0.1),
-        layers.Dense(PREDICTION_HORIZON * 2) # Output: Return and Range for each step
-    ])
-
-    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), loss='mse')
-    early_stop = callbacks.EarlyStopping(monitor='loss', patience=5, restore_best_weights=True)
-    model.fit(X, y, epochs=25, batch_size=32, verbose=0, callbacks=[early_stop])
+    X = data[feature_cols].values
+    y = data[target_names].values
+
+    model = RandomForestRegressor(
+        n_estimators=200, 
+        max_depth=20, 
+        min_samples_split=5,
+        min_samples_leaf=2,
+        max_features='sqrt',
+        n_jobs=-1, 
+        random_state=42
+    )
+    model.fit(X, y)
+    
+    # Calculate Residuals for Confidence Estimation
+    # (Using OOB or training residuals as a proxy for uncertainty)
+    predictions = model.predict(X)
+    residuals = y - predictions
+    residual_std = np.std(residuals, axis=0)
     
-    return model, scaler
+    return model, residual_std
 
-def get_prediction(df, model, scaler):
-    if model is None or scaler is None:
-        return []
+def get_prediction(df, model, residual_std):
+    if model is None or residual_std is None:
+        return [], []
     
     feature_cols = [
         'rsi', 'macd_hist', 'atr', 
@@ -171,95 +163,97 @@ def get_prediction(df, model, scaler):
         'hour_sin', 'hour_cos'
     ]
     
-    last_window = df.iloc[-LOOKBACK_WINDOW:][feature_cols].values
+    for lag in [1, 2, 3, 5, 8]:
+        feature_cols.extend([
+            f'rsi_lag{lag}', f'macd_hist_lag{lag}', 
+            f'log_ret_lag{lag}', f'vol_change_lag{lag}'
+        ])
     
-    if len(last_window) < LOOKBACK_WINDOW:
-        return []
+    last_row = df.iloc[[-1]][feature_cols]
     
-    last_window_scaled = scaler.transform(last_window)
-    last_window_reshaped = last_window_scaled.reshape(1, LOOKBACK_WINDOW, len(feature_cols))
-    
-    # Shape: (1, PREDICTION_HORIZON * 2)
-    raw_preds = model.predict(last_window_reshaped, verbose=0)[0]
+    if last_row.isnull().values.any():
+        return [], []
+
+    predicted_returns = model.predict(last_row.values)[0]
     
-    current_close = df.iloc[-1]['close']
+    current_price = df.iloc[-1]['close']
     current_time = int(df.iloc[-1]['time'])
+    current_atr = df.iloc[-1]['atr']
     
     pred_candles = []
+    confidence_data = []
     
-    # Initialize previous close as current real close for the chain
-    prev_close = current_close
-    
-    for i in range(PREDICTION_HORIZON):
-        # Extract pairs: [Return_1, Range_1, Return_2, Range_2, ...]
-        idx_ret = i * 2
-        idx_rng = i * 2 + 1
-        
-        pred_ret = raw_preds[idx_ret]
-        pred_rng = raw_preds[idx_rng]
+    prev_close = current_price
+
+    for i, pct_change in enumerate(predicted_returns):
+        future_time = current_time + ((i + 1) * 60)
         
-        # 1. Calculate Close
-        # Prediction is cumulative return from the START point (current_close)
-        # This is more stable than recursive step-by-step
-        future_close = current_close * (1 + pred_ret)
+        # Calculate predicted Close
+        future_close = current_price * (1 + pct_change)
         
-        # 2. Calculate Open
-        # Logic: Open of candle T is roughly Close of candle T-1
-        future_open = prev_close
+        # Construct Candle
+        # Open is previous candle's close
+        open_price = prev_close
+        close_price = future_close
         
-        # 3. Calculate High/Low using predicted Range
-        # Ensure Range isn't negative or impossibly small
-        candle_range_val = max(pred_rng * future_close, abs(future_close - future_open) * 1.1)
+        # Heuristic for High/Low to make it look like a candle
+        # Use ATR and some noise or just fixed ratio to visualize structure
+        # Here we use a fixed structure based on ATR to keep it clean but candle-like
+        half_range = current_atr * 0.4
         
-        # Center the range around the body
-        mid_point = (future_open + future_close) / 2
+        high_price = max(open_price, close_price) + half_range
+        low_price = min(open_price, close_price) - half_range
         
-        # Distribute range
-        future_high = mid_point + (candle_range_val / 2)
-        future_low = mid_point - (candle_range_val / 2)
+        pred_candles.append({
+            "time": future_time,
+            "open": float(open_price),
+            "high": float(high_price),
+            "low": float(low_price),
+            "close": float(close_price)
+        })
         
-        # Ensure High >= Max(O, C) and Low <= Min(O, C)
-        future_high = max(future_high, future_open, future_close)
-        future_low = min(future_low, future_open, future_close)
+        # Confidence Metric (Standard Deviation of Residuals at this step)
+        # We plot the error margin width relative to price
+        sigma = residual_std[i]
+        error_margin = future_close * sigma * 1.96 # 95% CI width approx
         
-        pred_candles.append({
-            "time": current_time + ((i + 1) * 60),
-            "open": float(future_open),
-            "high": float(future_high),
-            "low": float(future_low),
-            "close": float(future_close)
+        confidence_data.append({
+            "time": future_time,
+            "value": float(error_margin)
         })
         
         prev_close = future_close
     
-    return pred_candles
+    return pred_candles, confidence_data
 
 async def process_market_data():
     if not market_state['ready'] or not market_state['ohlc_history']:
         return {"error": "Initializing..."}
 
     df = calculate_indicators(market_state['ohlc_history'])
-    if df is None or len(df) < LOOKBACK_WINDOW + 50:
+    if df is None or len(df) < 100:
         return {"error": "Not enough data"}
 
+    # Retrain periodically
     if market_state['model'] is None or (time.time() - market_state['last_training_time'] > TRAIN_INTERVAL):
         try:
             loop = asyncio.get_running_loop()
-            model, scaler = await loop.run_in_executor(executor, train_model, df)
+            model, res_std = await loop.run_in_executor(executor, train_model, df)
             if model is not None:
                 market_state['model'] = model
-                market_state['scaler'] = scaler
+                market_state['model_residuals'] = res_std
                 market_state['last_training_time'] = time.time()
-                logging.info("CNN Model Retrained.")
         except Exception as e:
             logging.error(f"Training failed: {e}")
 
     predictions = []
+    confidence = []
     try:
-        predictions = get_prediction(df, market_state['model'], market_state['scaler'])
+        predictions, confidence = get_prediction(df, market_state['model'], market_state['model_residuals'])
     except Exception as e:
         logging.error(f"Prediction failed: {e}")
 
+    # Prepare display data
     df_clean = df.replace([np.inf, -np.inf], np.nan)
     cols_to_keep = ['time', 'open', 'high', 'low', 'close', 'volume', 'ema20', 'bb_upper', 'bb_lower', 'rsi', 'macd_hist']
     df_clean = df_clean[cols_to_keep].where(pd.notnull(df_clean), None)
@@ -277,23 +271,25 @@ async def process_market_data():
     return {
         "data": display_data,
         "prediction": predictions,
+        "confidence": confidence,
         "stats": {
             "price": last_close,
             "change": round(price_change, 2),
             "rsi": round(float(last_row.get('rsi', 0)), 1) if pd.notna(last_row.get('rsi')) else 0,
-            "macd": round(float(last_row.get('macd_hist', 0)), 2) if pd.notna(last_row.get('macd_hist')) else 0,
+            "macd": round(float(last_row.get('macd', 0)), 2) if pd.notna(last_row.get('macd')) else 0,
             "atr": round(float(last_row.get('atr', 0)), 2) if pd.notna(last_row.get('atr')) else 0,
             "volume": round(float(last_row.get('volume', 0)), 2) if pd.notna(last_row.get('volume')) else 0
         }
     }
 
+# --- HTML/JS ---
 HTML_PAGE = """
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>BTC/USD AI Candles</title>
+    <title>BTC/USD AI Predictor</title>
     <script src="https://unpkg.com/lightweight-charts@4.1.1/dist/lightweight-charts.standalone.production.js"></script>
     <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap" rel="stylesheet">
     <style>
@@ -367,7 +363,7 @@ HTML_PAGE = """
         }
         .chart-wrapper { position: relative; border-bottom: 1px solid rgba(255, 255, 255, 0.05); }
         #main-chart { flex: 5; }
-        #volume-chart { flex: 1; min-height: 60px; }
+        #volume-chart { flex: 1.5; min-height: 80px; }
         #osc-chart { flex: 1.5; min-height: 80px; }
         .chart-label {
             position: absolute; top: 12px; left: 16px; z-index: 10;
@@ -387,7 +383,6 @@ HTML_PAGE = """
             border-top-color: #00ff88; border-radius: 50%; animation: spin 1s linear infinite;
         }
         @keyframes spin { to { transform: rotate(360deg); } }
-        .loading-text { margin-top: 20px; font-size: 14px; color: #666; }
         .prediction-badge {
             position: absolute; top: 12px; right: 16px;
             background: rgba(191, 90, 242, 0.15); border: 1px solid rgba(191, 90, 242, 0.3);
@@ -429,29 +424,28 @@ HTML_PAGE = """
         <div class="indicator-group"><span class="indicator-label">BB Upper</span><span id="bb-upper" class="indicator-value" style="color: #26a69a">--</span></div>
         <div class="indicator-group"><span class="indicator-label">BB Lower</span><span id="bb-lower" class="indicator-value" style="color: #ef5350">--</span></div>
         <div class="indicator-group"><span class="indicator-label">MACD</span><span id="macd-val" class="indicator-value">--</span></div>
-        <div class="indicator-group"><span class="indicator-label">Volume</span><span id="vol-val" class="indicator-value" style="color: #888">--</span></div>
     </div>
     <div class="charts-container">
         <div class="loading-overlay" id="loading">
             <div class="loader"></div>
-            <div class="loading-text">Loading market data...</div>
         </div>
         <div id="main-chart" class="chart-wrapper">
             <div class="chart-label">
-                <span><div class="dot" style="background: #00ff88"></div>History</span>
-                <span><div class="dot" style="background: #00b8d4"></div>AI Prediction</span>
-                <span><div class="dot" style="background: #2962FF"></div>EMA 20</span>
-                <span><div class="dot" style="background: #26a69a; opacity: 0.5"></div>Bollinger</span>
+                <span><div class="dot" style="background: #00ff88"></div>Price</span>
+                <span><div class="dot" style="background: #bf5af2"></div>AI Prediction</span>
             </div>
-            <div class="prediction-badge">CNN: 100 Candles</div>
+            <div class="prediction-badge">Forecast: 100 Candles</div>
         </div>
         <div id="volume-chart" class="chart-wrapper">
-            <div class="chart-label"><span><div class="dot" style="background: #5c6bc0"></div>Volume</span></div>
+            <div class="chart-label">
+                <span><div class="dot" style="background: #5c6bc0"></div>Volume</span>
+                <span><div class="dot" style="background: #ff9f43"></div>AI Uncertainty (±$)</span>
+            </div>
         </div>
         <div id="osc-chart" class="chart-wrapper">
             <div class="chart-label">
                 <span><div class="dot" style="background: #9C27B0"></div>RSI</span>
-                <span><div class="dot" style="background: #26a69a"></div>MACD Hist</span>
+                <span><div class="dot" style="background: #26a69a"></div>MACD</span>
             </div>
         </div>
     </div>
@@ -478,27 +472,42 @@ document.addEventListener('DOMContentLoaded', () => {
     const volChart = LightweightCharts.createChart(volEl, chartOptions);
     const oscChart = LightweightCharts.createChart(oscEl, chartOptions);
 
-    // Historic Data Series
+    // Main Chart Series
     const candles = mainChart.addCandlestickSeries({
         upColor: '#00ff88', downColor: '#ff4757',
         borderUpColor: '#00ff88', borderDownColor: '#ff4757',
         wickUpColor: '#00ff88', wickDownColor: '#ff4757'
     });
 
-    // Prediction Data Series (Distinct Colors)
-    const predCandles = mainChart.addCandlestickSeries({
-        upColor: '#00b8d4', downColor: '#aa00ff', // Cyan for up, Purple for down
-        borderUpColor: '#00b8d4', borderDownColor: '#aa00ff',
-        wickUpColor: '#00b8d4', wickDownColor: '#aa00ff'
-    });
-
     const ema = mainChart.addLineSeries({ color: '#2962FF', lineWidth: 2, crosshairMarkerVisible: false });
     const bbUpper = mainChart.addLineSeries({ color: 'rgba(38, 166, 154, 0.4)', lineWidth: 1, crosshairMarkerVisible: false });
     const bbLower = mainChart.addLineSeries({ color: 'rgba(239, 83, 80, 0.4)', lineWidth: 1, crosshairMarkerVisible: false });
 
+    // AI Prediction Series (Candles)
+    const predCandles = mainChart.addCandlestickSeries({
+        upColor: 'rgba(191, 90, 242, 0.8)', downColor: 'rgba(191, 90, 242, 0.8)',
+        borderUpColor: '#bf5af2', borderDownColor: '#bf5af2',
+        wickUpColor: '#bf5af2', wickDownColor: '#bf5af2'
+    });
+
+    // Volume Chart Series
     const volumeSeries = volChart.addHistogramSeries({ priceFormat: { type: 'volume' }, priceScaleId: '' });
     volChart.priceScale('').applyOptions({ scaleMargins: { top: 0.1, bottom: 0 } });
 
+    // Confidence/Uncertainty Series (Near Volume)
+    const confidenceSeries = volChart.addLineSeries({
+        color: '#ff9f43',
+        lineWidth: 2,
+        priceScaleId: 'confidence',
+        lineStyle: LightweightCharts.LineStyle.Solid
+    });
+    // Position the confidence scale to not overlap heavily with volume (overlay mode)
+    volChart.priceScale('confidence').applyOptions({
+        scaleMargins: { top: 0.1, bottom: 0.7 }, // Keep it at top of volume pane
+        visible: true
+    });
+
+    // Oscillator Chart Series
     const rsi = oscChart.addLineSeries({ color: '#9C27B0', lineWidth: 2, priceScaleId: 'rsi' });
     oscChart.priceScale('rsi').applyOptions({ scaleMargins: { top: 0.1, bottom: 0.1 } });
 
@@ -544,7 +553,6 @@ document.addEventListener('DOMContentLoaded', () => {
                 macdEl.textContent = macdVal.toFixed(2);
                 macdEl.style.color = macdVal >= 0 ? '#26a69a' : '#ef5350';
             }
-            document.getElementById('vol-val').textContent = lastData.volume ? lastData.volume.toFixed(2) : '--';
         }
     }
 
@@ -569,6 +577,7 @@ document.addEventListener('DOMContentLoaded', () => {
                 const candleData = d.filter(x => x && x.time && x.open).map(x => ({
                     time: x.time, open: x.open, high: x.high, low: x.low, close: x.close
                 }));
+                
                 if (candleData.length > 0) {
                     candles.setData(candleData);
                     ema.setData(safeMap(d, 'ema20'));
@@ -582,10 +591,17 @@ document.addEventListener('DOMContentLoaded', () => {
                         time: x.time, value: x.macd_hist, color: x.macd_hist >= 0 ? '#26a69a' : '#ef5350'
                     })));
                     
+                    // Update Prediction Candles
                     if (payload.prediction && payload.prediction.length > 0) {
+                        // Prediction data is already in OHLC format
                         predCandles.setData(payload.prediction);
                     }
                     
+                    // Update Confidence Metric (Volume Pane)
+                    if (payload.confidence && payload.confidence.length > 0) {
+                        confidenceSeries.setData(payload.confidence);
+                    }
+
                     updateStats(payload.stats, d[d.length - 1]);
                     if (!hasData) {
                         hasData = true;
@@ -689,6 +705,7 @@ async def broadcast_worker():
             payload = await process_market_data()
             if payload and "data" in payload:
                 msg = json.dumps(payload)
+                # Iterate over a copy to avoid RuntimeError if set size changes
                 current_clients = connected_clients.copy()
                 disconnected = set()
                 for ws in current_clients: