Update app.py

app.py

@@ -8,12 +8,12 @@ import numpy as np
 from aiohttp import web
 import websockets
 from sklearn.ensemble import RandomForestRegressor
-from sklearn.model_selection import train_test_split
 
+# Configuration
 SYMBOL_KRAKEN = "BTC/USD"
 PORT = 7860
-BROADCAST_RATE = 1.0 # Slowed down slightly for ML inference time
-PREDICTION_HORIZON = 100 # Predict next 100 minutes
+BROADCAST_RATE = 1.0 
+PREDICTION_HORIZON = 100 
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
 
@@ -83,22 +83,26 @@ def train_model(df):
     logging.info("Training ML Model...")
     
     # 1. Prepare Features (X)
-    # We use the indicators we calculated as features
     feature_cols = ['close', 'ema', 'bb_upper', 'bb_lower', 'rsi', 'macd', 'stoch_k', 'atr', 'obv', 'vwap']
     
-    # Drop NaNs created by indicators
+    # Create a clean copy for training data
     data = df.dropna().copy()
     
-    # Create Targets (y)
-    # y is the Close price shifted backwards by 1 to 100 steps
-    # If t is now, we want to predict Close at t+1, t+2... t+100
+    # 2. Create Targets (y) - OPTIMIZED to fix Fragmentation Warning
+    # Instead of adding columns in a loop, we create a dict and concat once
+    future_shifts = {}
     targets = []
+    
     for i in range(1, PREDICTION_HORIZON + 1):
         col_name = f'target_{i}'
-        data[col_name] = data['close'].shift(-i)
+        future_shifts[col_name] = data['close'].shift(-i)
         targets.append(col_name)
+        
+    # Concatenate all target columns at once
+    target_df = pd.DataFrame(future_shifts, index=data.index)
+    data = pd.concat([data, target_df], axis=1)
 
-    # Drop the end rows where we don't have future data yet
+    # Drop rows where we don't have future data (the last 100 rows)
     data = data.dropna()
 
     if len(data) < 100:
@@ -109,7 +113,6 @@ def train_model(df):
     y = data[targets].values
 
     # Train Random Forest
-    # n_estimators=50 is kept low for speed in this demo. Increase for accuracy.
     model = RandomForestRegressor(n_estimators=50, max_depth=10, n_jobs=-1, random_state=42)
     model.fit(X, y)
     
@@ -127,14 +130,14 @@ def get_prediction(df, model):
     if last_row.isnull().values.any(): return []
 
     # Predict
-    prediction = model.predict(last_row.values)[0] # Returns array of 100 values
+    prediction = model.predict(last_row.values)[0] 
     
     # Format for frontend
     current_time = int(df.iloc[-1]['time'])
     pred_data = []
     for i, price in enumerate(prediction):
         pred_data.append({
-            "time": current_time + ((i + 1) * 60), # Add minutes
+            "time": current_time + ((i + 1) * 60), 
             "value": float(price)
         })
     
@@ -149,7 +152,7 @@ def process_market_data():
     if df is None or len(df) < 50: return {"error": "Not enough data"}
 
     # 2. Train Model (Periodically)
-    # Train initially or every 15 minutes to adapt to new trends
+    # Train initially or every 15 minutes (900 seconds)
     if market_state['model'] is None or (time.time() - market_state['last_training_time'] > 900):
         market_state['model'] = train_model(df)
         market_state['last_training_time'] = time.time()
@@ -222,7 +225,6 @@ HTML_PAGE = f"""
         const mainChart = LightweightCharts.createChart(mainEl, commonOpts);
         const candles = mainChart.addCandlestickSeries({{ upColor: '#00ff9d', downColor: '#ff3b3b', borderVisible: false }});
         const ema = mainChart.addLineSeries({{ color: '#2962FF', lineWidth: 1 }});
-        // Prediction Line
         const predLine = mainChart.addLineSeries({{ color: '#bf5af2', lineWidth: 2, lineStyle: 2, title: 'AI Forecast' }});
 
         const oscChart = LightweightCharts.createChart(oscEl, commonOpts);
@@ -258,7 +260,6 @@ HTML_PAGE = f"""
                 ema.setData(mapData('ema'));
                 rsi.setData(mapData('rsi'));
                 
-                // Prediction Data (Future)
                 if(payload.prediction && payload.prediction.length > 0) {{
                     predLine.setData(payload.prediction);
                 }}
@@ -283,8 +284,6 @@ HTML_PAGE = f"""
 async def kraken_worker():
     global market_state
     try:
-        # 1. REST Snapshot (Get MORE data for training)
-        # 720 is roughly Kraken's max per request for 1m timeframe
         async with aiohttp.ClientSession() as session:
             url = "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1" 
             async with session.get(url) as response:
@@ -294,7 +293,6 @@ async def kraken_worker():
                         for key in data['result']:
                             if key != 'last':
                                 raw = data['result'][key]
-                                # Keep last 720 candles for training
                                 market_state['ohlc_history'] = [
                                     {
                                         'time': int(c[0]), 
@@ -311,7 +309,7 @@ async def kraken_worker():
     except Exception as e:
         logging.error(f"Init Error: {e}")
 
-    # 2. WebSocket Stream
+    # WebSocket Stream
     while True:
         try:
             async with websockets.connect("wss://ws.kraken.com/v2") as ws:
@@ -347,7 +345,6 @@ async def kraken_worker():
                                             'close': price, 
                                             'volume': vol
                                         })
-                                        # Keep buffer size larger for ML
                                         if len(market_state['ohlc_history']) > 800: 
                                             market_state['ohlc_history'].pop(0)
                             except: pass
@@ -379,7 +376,6 @@ async def kraken_worker():
 async def broadcast_worker():
     while True:
         if connected_clients and market_state['ready']:
-            # This calculation might take 100-200ms depending on CPU
             payload = process_market_data()
             if payload and "data" in payload:
                 msg = json.dumps(payload)