Update app.py

app.py

@@ -6,8 +6,10 @@ import math
 import aiohttp
 import pandas as pd
 import numpy as np
+import tensorflow as tf
 from aiohttp import web
-from sklearn.ensemble import RandomForestRegressor
+from tensorflow.keras import layers, models, callbacks
+from sklearn.preprocessing import StandardScaler
 from concurrent.futures import ThreadPoolExecutor
 
 SYMBOL_KRAKEN = "BTC/USD"
@@ -15,7 +17,8 @@ PORT = 7860
 BROADCAST_RATE = 1.0
 PREDICTION_HORIZON = 100
 MAX_HISTORY = 5000
-TRAIN_INTERVAL = 300
+TRAIN_INTERVAL = 600
+LOOKBACK_WINDOW = 60
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
 
@@ -23,7 +26,8 @@ market_state = {
     "ohlc_history": [],
     "ready": False,
     "model": None,
-    "model_residuals": None,
+    "scaler": None,
+    "model_residuals": 0.0,
     "last_training_time": 0,
     "last_price": 0,
     "price_change": 0
@@ -33,7 +37,7 @@ connected_clients = set()
 executor = ThreadPoolExecutor(max_workers=1)
 
 def calculate_indicators(candles):
-    if len(candles) < 100:
+    if len(candles) < LOOKBACK_WINDOW + PREDICTION_HORIZON:
         return None
     
     df = pd.DataFrame(candles)
@@ -77,16 +81,17 @@ def calculate_indicators(candles):
     df['hour_sin'] = np.sin(2 * np.pi * df['datetime'].dt.hour / 24)
     df['hour_cos'] = np.cos(2 * np.pi * df['datetime'].dt.hour / 24)
 
-    for lag in [1, 2, 3, 5, 8]:
-        df[f'rsi_lag{lag}'] = df['rsi'].shift(lag)
-        df[f'macd_hist_lag{lag}'] = df['macd_hist'].shift(lag)
-        df[f'log_ret_lag{lag}'] = df['log_ret'].shift(lag)
-        df[f'vol_change_lag{lag}'] = df['vol_change'].shift(lag)
+    return df.dropna()
 
-    return df
+def create_sequences(data, target_data, window_size, horizon):
+    X, y = [], []
+    for i in range(len(data) - window_size - horizon + 1):
+        X.append(data[i:(i + window_size)])
+        y.append(target_data[i + window_size : i + window_size + horizon])
+    return np.array(X), np.array(y)
 
 def train_model(df):
-    logging.info(f"Training ML Model on {len(df)} candles...")
+    logging.info(f"Training CNN Model on {len(df)} candles...")
     
     feature_cols = [
         'rsi', 'macd_hist', 'atr', 
@@ -96,50 +101,65 @@ def train_model(df):
         'hour_sin', 'hour_cos'
     ]
     
-    for lag in [1, 2, 3, 5, 8]:
-        feature_cols.extend([
-            f'rsi_lag{lag}', f'macd_hist_lag{lag}', 
-            f'log_ret_lag{lag}', f'vol_change_lag{lag}'
-        ])
+    data_features = df[feature_cols].values
     
-    data = df.dropna().copy()
+    scaler = StandardScaler()
+    data_scaled = scaler.fit_transform(data_features)
     
-    target_cols_dict = {}
-    target_names = []
+    close_prices = df['close'].values
+    returns_future = []
     
-    for i in range(1, PREDICTION_HORIZON + 1):
-        col_name = f'target_return_{i}'
-        target_cols_dict[col_name] = (data['close'].shift(-i) - data['close']) / data['close']
-        target_names.append(col_name)
+    for i in range(len(close_prices) - PREDICTION_HORIZON):
+        current_price = close_prices[i]
+        future_prices = close_prices[i+1 : i+1+PREDICTION_HORIZON]
+        pct_change = (future_prices - current_price) / current_price
+        returns_future.append(pct_change)
     
-    targets_df = pd.DataFrame(target_cols_dict, index=data.index)
-    data = pd.concat([data, targets_df], axis=1).dropna()
-
-    if len(data) < 200:
-        return None, None
-
-    X = data[feature_cols].values
-    y = data[target_names].values
+    returns_future = np.array(returns_future)
+    
+    X = []
+    y = []
+    
+    valid_length = len(returns_future) - LOOKBACK_WINDOW
+    if valid_length <= 0:
+        return None, None, None
 
-    model = RandomForestRegressor(
-        n_estimators=200, 
-        max_depth=25, 
-        min_samples_split=5,
-        min_samples_leaf=2,
-        max_features='sqrt',
-        n_jobs=-1, 
-        random_state=42
-    )
-    model.fit(X, y)
+    for i in range(valid_length):
+        X.append(data_scaled[i : i + LOOKBACK_WINDOW])
+        y.append(returns_future[i + LOOKBACK_WINDOW - 1]) 
+        
+    X = np.array(X)
+    y = np.array(y)
     
-    predictions = model.predict(X)
+    if len(X) < 100:
+        return None, None, None
+
+    model = models.Sequential([
+        layers.Input(shape=(LOOKBACK_WINDOW, len(feature_cols))),
+        layers.Conv1D(filters=64, kernel_size=3, activation='relu', padding='same'),
+        layers.MaxPooling1D(pool_size=2),
+        layers.Dropout(0.2),
+        layers.Conv1D(filters=32, kernel_size=3, activation='relu', padding='same'),
+        layers.GlobalAveragePooling1D(),
+        layers.Dense(64, activation='relu'),
+        layers.Dropout(0.1),
+        layers.Dense(PREDICTION_HORIZON)
+    ])
+
+    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), loss='mse')
+    
+    early_stop = callbacks.EarlyStopping(monitor='loss', patience=5, restore_best_weights=True)
+    
+    model.fit(X, y, epochs=20, batch_size=32, verbose=0, callbacks=[early_stop])
+    
+    predictions = model.predict(X, verbose=0)
     residuals = y - predictions
-    residual_std = np.std(residuals, axis=0)
+    residual_std = np.std(residuals) 
     
-    return model, residual_std
+    return model, scaler, residual_std
 
-def get_prediction(df, model, residual_std):
-    if model is None or residual_std is None:
+def get_prediction(df, model, scaler, residual_std):
+    if model is None or scaler is None:
         return []
     
     feature_cols = [
@@ -150,32 +170,34 @@ def get_prediction(df, model, residual_std):
         'hour_sin', 'hour_cos'
     ]
     
-    for lag in [1, 2, 3, 5, 8]:
-        feature_cols.extend([
-            f'rsi_lag{lag}', f'macd_hist_lag{lag}', 
-            f'log_ret_lag{lag}', f'vol_change_lag{lag}'
-        ])
-    
-    last_row = df.iloc[[-1]][feature_cols]
+    last_window = df.iloc[-LOOKBACK_WINDOW:][feature_cols].values
     
-    if last_row.isnull().values.any():
+    if len(last_window) < LOOKBACK_WINDOW:
         return []
-
-    predicted_returns = model.predict(last_row.values)[0]
+    
+    last_window_scaled = scaler.transform(last_window)
+    last_window_reshaped = last_window_scaled.reshape(1, LOOKBACK_WINDOW, len(feature_cols))
+    
+    predicted_returns = model.predict(last_window_reshaped, verbose=0)[0]
     
     current_price = df.iloc[-1]['close']
     current_time = int(df.iloc[-1]['time'])
     
     pred_data = []
     confidence_multiplier = 1.96 
-
+    
+    time_step = 0
+    accumulated_variance = 0.0
+    
     for i, pct_change in enumerate(predicted_returns):
         future_price = current_price * (1 + pct_change)
         
-        sigma = residual_std[i]
-        upper_bound = future_price * (1 + (sigma * confidence_multiplier))
-        lower_bound = future_price * (1 - (sigma * confidence_multiplier))
-
+        accumulated_variance += (residual_std ** 2)
+        current_std = np.sqrt(accumulated_variance) / np.sqrt(i + 1) * (i + 1) * 0.5 
+        
+        upper_bound = future_price * (1 + (residual_std * confidence_multiplier))
+        lower_bound = future_price * (1 - (residual_std * confidence_multiplier))
+        
         pred_data.append({
             "time": current_time + ((i + 1) * 60),
             "value": float(future_price),
@@ -190,23 +212,25 @@ async def process_market_data():
         return {"error": "Initializing..."}
 
     df = calculate_indicators(market_state['ohlc_history'])
-    if df is None or len(df) < 100:
+    if df is None or len(df) < LOOKBACK_WINDOW + 50:
         return {"error": "Not enough data"}
 
     if market_state['model'] is None or (time.time() - market_state['last_training_time'] > TRAIN_INTERVAL):
         try:
             loop = asyncio.get_running_loop()
-            model, res_std = await loop.run_in_executor(executor, train_model, df)
+            model, scaler, res_std = await loop.run_in_executor(executor, train_model, df)
             if model is not None:
                 market_state['model'] = model
-                market_state['model_residuals'] = res_std
+                market_state['scaler'] = scaler
+                market_state['model_residuals'] = float(res_std)
                 market_state['last_training_time'] = time.time()
+                logging.info(f"Model retrained. Residual Std: {market_state['model_residuals']:.5f}")
         except Exception as e:
             logging.error(f"Training failed: {e}")
 
     predictions = []
     try:
-        predictions = get_prediction(df, market_state['model'], market_state['model_residuals'])
+        predictions = get_prediction(df, market_state['model'], market_state['scaler'], market_state['model_residuals'])
     except Exception as e:
         logging.error(f"Prediction failed: {e}")
 
@@ -231,7 +255,7 @@ async def process_market_data():
             "price": last_close,
             "change": round(price_change, 2),
             "rsi": round(float(last_row.get('rsi', 0)), 1) if pd.notna(last_row.get('rsi')) else 0,
-            "macd": round(float(last_row.get('macd', 0)), 2) if pd.notna(last_row.get('macd')) else 0,
+            "macd": round(float(last_row.get('macd_hist', 0)), 2) if pd.notna(last_row.get('macd_hist')) else 0,
             "atr": round(float(last_row.get('atr', 0)), 2) if pd.notna(last_row.get('atr')) else 0,
             "volume": round(float(last_row.get('volume', 0)), 2) if pd.notna(last_row.get('volume')) else 0
         }
@@ -243,7 +267,7 @@ HTML_PAGE = """
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>BTC/USD AI Predictor</title>
+    <title>BTC/USD AI Predictor (CNN)</title>
     <script src="https://unpkg.com/lightweight-charts@4.1.1/dist/lightweight-charts.standalone.production.js"></script>
     <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap" rel="stylesheet">
     <style>
@@ -391,7 +415,7 @@ HTML_PAGE = """
                 <span><div class="dot" style="background: #00ff88"></div>Price</span>
                 <span><div class="dot" style="background: #2962FF"></div>EMA 20</span>
                 <span><div class="dot" style="background: #26a69a; opacity: 0.5"></div>Bollinger</span>
-                <span><div class="dot" style="background: #bf5af2"></div>AI + 95% Conf</span>
+                <span><div class="dot" style="background: #bf5af2"></div>CNN Forecast</span>
             </div>
             <div class="prediction-badge">AI Forecast: 100 candles</div>
         </div>