Upload main.py

main.py

@@ -1,294 +1,335 @@
-"""AlphaForge - Orchestrator wiring all modules together."""
-import argparse
-import os
-import json
-import numpy as np
-import pandas as pd
-import torch
-import warnings
+"""AlphaForge - Complete Quantitative Trading System v2.0
+
+Improved features:
+  - Real-time data streaming (Alpaca, Polygon, Yahoo)
+  - Advanced feature engineering (microstructure, macro, stat-arb, regime)
+  - Online learning with drift detection
+  - News/sentiment streaming
+  - Order flow estimation
+
+Usage:
+    python main.py --mode train --tickers SPY QQQ AAPL MSFT
+    python main.py --mode backtest --start 2020-01-01 --end 2024-01-01
+    python main.py --mode realtime --source yahoo --tickers SPY QQQ
+    python main.py --mode options
+"""
+import argparse, numpy as np, pandas as pd, torch, os, json, warnings
 warnings.filterwarnings('ignore')
 
-# Core modules
 from market_data import MarketDataPipeline
 from alpha_model import AlphaEnsemble
 from sentiment_model import SentimentAlphaModel
 from volatility_model import VolatilityEngine
 from portfolio_optimizer import PortfolioOptimizer
+from options_pricer import MLOptionsPricer
 from backtest_engine import BacktestEngine, compute_information_coefficient, RegimeDetector
 
-# Advanced modules
-from meta_model import MetaModel
-from regime_detector import RegimeDetectorHMM
-from risk_engine import RiskEngine, DrawdownControl
-from factor_decomposition import FactorDecomposition
-from online_learning import OnlineLearner, AdaptiveEnsemble
-from explainability import ExplainabilityLayer
-from anomaly_detector import AnomalyDetector
-from stress_test import StressTestEngine
-from bayesian_layer import BayesianForecaster, BayesianOptimizer
-from hedging_engine import DynamicHedgingEngine
-from strategy_ensemble import StrategyEnsemble
+# v2 imports
+from advanced_features_part1 import MicrostructureFeatures, CrossSectionalFeatures
+from macro_features import MacroFeatures
+from regime_features import RegimeFeatures
+from technical_indicators import AdvancedTechnical
+from stat_arb_features import StatArbFeatures
+from online_learning import OnlineLearner, DriftDetector
+from realtime_data import RealtimeFeatureEngine, LiveDataBuffer, OrderFlowEstimator, NewsStreamAggregator
 
 
-def create_feature_names():
-    """Create feature name list for explainability."""
-    return ['return_1d','return_5d','return_10d','return_21d','return_63d',
-            'rvol_5d','rvol_21d','rvol_63d',
-            'sma_5d','sma_10d','sma_20d','sma_50d','sma_200d',
-            'rsi_14','macd','macd_signal','bb_position',
-            'volume_sma_ratio','volume_change','intraday_range','open_gap']
+def parse_args():
+    p = argparse.ArgumentParser(description='AlphaForge v2.0')
+    p.add_argument('--mode', default='backtest', choices=['train','backtest','realtime','options'])
+    p.add_argument('--tickers', nargs='+', default=['SPY','QQQ','AAPL','MSFT','GOOGL','AMZN','META','NVDA','TSLA','JPM'])
+    p.add_argument('--start', default='2020-01-01')
+    p.add_argument('--end', default='2024-01-01')
+    p.add_argument('--lookback', type=int, default=60)
+    p.add_argument('--horizon', type=int, default=5)
+    p.add_argument('--epochs', type=int, default=50)
+    p.add_argument('--device', default='cpu')
+    p.add_argument('--capital', type=float, default=1_000_000)
+    p.add_argument('--output', default='results/')
+    p.add_argument('--source', default='yahoo', choices=['yahoo','alpaca','polygon'])
+    p.add_argument('--api-key', default='')
+    p.add_argument('--secret-key', default='')
+    p.add_argument('--advanced-features', action='store_true', help='Use advanced feature engineering')
+    p.add_argument('--include-macro', action='store_true', help='Include FRED macro data')
+    p.add_argument('--include-sentiment', action='store_true', help='Include FinBERT sentiment')
+    p.add_argument('--online-learning', action='store_true', help='Enable online drift detection')
+    return p.parse_args()
+
 
+def build_advanced_features(data, include_macro=True):
+    """Build 90+ feature matrix using advanced feature engineering"""
+    all_features = []
+    for ticker, df in data.items():
+        close = df['Close'].values.flatten()
+        high = df['High'].values.flatten()
+        low = df['Low'].values.flatten()
+        volume = df['Volume'].values.flatten()
+        close_s = pd.Series(close, index=df.index)
+        high_s = pd.Series(high, index=df.index)
+        low_s = pd.Series(low, index=df.index)
+        vol_s = pd.Series(volume, index=df.index)
+        
+        features = pd.DataFrame(index=df.index)
+        features['ticker'] = ticker
+        features['close'] = close
+        
+        # Microstructure
+        micro = MicrostructureFeatures.compute_all(close_s, high_s, low_s, vol_s)
+        for c in micro.columns:
+            features[f'micro_{c}'] = micro[c]
+        
+        # Regime
+        returns = close_s.pct_change().fillna(0)
+        vol_regime = RegimeFeatures.volatility_regime(returns)
+        liq_regime = RegimeFeatures.liquidity_regime(vol_s, close_s)
+        trend_regime = RegimeFeatures.trend_regime(close_s)
+        for df_r in [vol_regime, liq_regime, trend_regime]:
+            for c in df_r.columns:
+                features[c] = df_r[c]
+        
+        # Advanced technicals
+        ichimoku = AdvancedTechnical.ichimoku(close_s, high_s, low_s)
+        supertrend = AdvancedTechnical.supertrend(close_s, high_s, low_s)
+        vp = AdvancedTechnical.volume_profile(close_s, vol_s, high_s, low_s)
+        keltner = AdvancedTechnical.keltner_channels(close_s, high_s, low_s)
+        for df_t in [ichimoku, supertrend, vp, keltner]:
+            for c in df_t.columns:
+                features[f'ta_{c}'] = df_t[c]
+        
+        all_features.append(features)
+    
+    features_df = pd.concat(all_features, axis=0)
+    
+    # Macro overlay
+    if include_macro:
+        macro = MacroFeatures._synthetic_macro(str(features_df.index[0])[:10], str(features_df.index[-1])[:10])
+        for c in macro.columns:
+            features_df[f'macro_{c}'] = macro[c].reindex(features_df.index).ffill()
+    
+    # Z-score per ticker
+    numeric_cols = [c for c in features_df.columns if c not in ['ticker','close']]
+    for ticker in features_df['ticker'].unique():
+        mask = features_df['ticker'] == ticker
+        for col in numeric_cols:
+            s = features_df.loc[mask, col]
+            roll_mean = s.rolling(42).mean()
+            roll_std = s.rolling(42).std().replace(0, 1)
+            features_df.loc[mask, col] = (s - roll_mean) / roll_std
+    
+    return features_df.replace([np.inf, -np.inf], 0).fillna(0)
 
-def run_full_pipeline(args):
-    """Run the complete AlphaForge pipeline."""
-    print("=" * 70)
-    print("   🏦  A L P H A F O R G E  -  Autonomous Quant Fund OS")
-    print("=" * 70)
-    
-    # ---------- 1. DATA PIPELINE ----------
-    print("\n[1/12] Fetching market data...")
+
+def run_backtest(args):
+    """Run full pipeline backtest"""
+    print("=" * 60)
+    print("ALPHA FORGE v2.0 - Full Pipeline Backtest")
+    print("=" * 60)
+    
+    # Fetch data
     pipeline = MarketDataPipeline(args.tickers, args.start, args.end)
     data = pipeline.fetch_data()
-    features_df = pipeline.create_feature_matrix()
-    X, y, tickers_arr, dates = pipeline.create_sequences(features_df, lookback=args.lookback, forecast_horizon=args.horizon)
     
+    # Build features
+    print("\n[1/6] Building features...")
+    if args.advanced_features:
+        features_df = build_advanced_features(data, include_macro=args.include_macro)
+        print(f"  Advanced features: {features_df.shape[1] - 2} columns")
+    else:
+        features_df = pipeline.create_feature_matrix()
+    
+    X, y, tickers_arr, dates = pipeline.create_sequences(features_df, args.lookback, args.horizon)
+    print(f"  Dataset: {len(X)} samples, {X.shape[2]} features")
+    
+    # Sentiment
+    sentiment_alpha = None
+    if args.include_sentiment:
+        print("\n[2/6] Running sentiment analysis...")
+        sentiment_model = SentimentAlphaModel(device=args.device)
+        dates_idx = pd.date_range(args.start, args.end, freq='B')
+        news_df = sentiment_model.generate_synthetic_news(args.tickers, dates_idx[:60], n_news_per_day=2)
+        sentiment_df = sentiment_model.generate_sentiment_alpha(news_df)
+        print(f"  Sentiment scores: {len(sentiment_df)} entries")
+    
+    # Train alpha model
+    print("\n[3/6] Training Alpha Model...")
     n = len(X)
-    train_end = int(n * 0.70)
+    train_end = int(n * 0.7)
     val_end = int(n * 0.85)
+    
     X_train, y_train = X[:train_end], y[:train_end]
     X_val, y_val = X[train_end:val_end], y[train_end:val_end]
     X_test, y_test = X[val_end:], y[val_end:]
-    tickers_test = tickers_arr[val_end:]
-    dates_test = dates[val_end:]
-    print(f"   Samples: {len(X):,} (train: {len(X_train):,}, val: {len(X_val):,}, test: {len(X_test):,})")
     
-    # ---------- 2. ALPHA MODEL ----------
-    print("\n[2/12] Training Alpha Model ensemble (LSTM + Transformer + XGBoost)...")
     ensemble = AlphaEnsemble(input_size=X.shape[2], seq_len=args.lookback, device=args.device)
-    alpha_metrics = ensemble.fit(X_train, y_train, X_val, y_val, epochs=args.epochs, batch_size=64, lr=1e-4)
+    metrics = ensemble.fit(X_train, y_train, X_val, y_val, epochs=args.epochs, batch_size=64, lr=1e-4)
     
-    # Generate base predictions
-    lstm_pred = ensemble.lstm(torch.FloatTensor(X_test).to(ensemble.device)).cpu().detach().numpy().flatten()
-    trans_pred = ensemble.transformer(torch.FloatTensor(X_test).to(ensemble.device)).cpu().detach().numpy().flatten()
-    xgb_pred = ensemble.xgboost.predict(X_test)
     alpha_pred = ensemble.predict(X_test)
-    
-    # ---------- 3. SENTIMENT MODEL ----------
-    print("\n[3/12] Running sentiment analysis (FinBERT)...")
-    sentiment_model = SentimentAlphaModel(device=args.device)
-    news_data = sentiment_model.generate_synthetic_news(args.tickers[:10], pd.DatetimeIndex(dates_test[:100]))
-    sentiment_df = sentiment_model.generate_sentiment_alpha(news_data, window=5)
-    sentiment_preds = np.zeros(len(y_test))
-    print(f"   Analyzed {len(news_data)} synthetic news items")
-    
-    # ---------- 4. META-MODEL ----------
-    print("\n[4/12] Training Meta-Model (learns which signal to trust)...")
-    meta = MetaModel(base_models=['lstm','transformer','xgboost','sentiment'], device=args.device)
-    predictions_train = {
-        'lstm': ensemble.lstm(torch.FloatTensor(X_train[:1000]).to(ensemble.device)).cpu().detach().numpy().flatten(),
-        'transformer': ensemble.transformer(torch.FloatTensor(X_train[:1000]).to(ensemble.device)).cpu().detach().numpy().flatten(),
-        'xgboost': ensemble.xgboost.predict(X_train[:1000]),
-        'sentiment': np.zeros(1000)
-    }
-    meta.fit(predictions_train, y_train[:1000])
-    
-    predictions_test = {
-        'lstm': lstm_pred, 'transformer': trans_pred,
-        'xgboost': xgb_pred, 'sentiment': sentiment_preds
-    }
-    meta_pred = meta.predict_meta(predictions_test)
-    meta_ic = compute_information_coefficient(pd.Series(meta_pred), pd.Series(y_test), by_date=False)
-    print(f"   Meta-model IC: {meta_ic['mean_ic']:.4f}")
-    
-    # ---------- 5. REGIME DETECTION ----------
-    print("\n[5/12] Detecting market regimes (HMM)...")
-    all_returns = {}
+    ic = compute_information_coefficient(pd.Series(alpha_pred), pd.Series(y_test), by_date=False)
+    print(f"  Test IC: {ic['mean_ic']:.4f}")
+    
+    # Online learning check
+    if args.online_learning:
+        print("\n[4/6] Checking for drift...")
+        detector = DriftDetector()
+        detector.set_reference(X_train, 'features')
+        drift_result = detector.detect_ks(X_test[:500], 'features')
+        print(f"  Drift: {drift_result['n_features_drifted']}/{drift_result['total_features']} features shifted")
+        if drift_result['drift']:
+            learner = OnlineLearner(ensemble.lstm)
+            adapt_result = learner.check_and_adapt(X_test[:500], y_test[:500])
+            print(f"  Adaptation: {adapt_result['adapted']}")
+    
+    # Volatility
+    print("\n[5/6] Building covariance...")
+    vol_engine = VolatilityEngine()
+    returns_dict = {}
     for ticker in args.tickers:
         if ticker in data:
-            c = data[ticker]['Close'].values.flatten()
-            all_returns[ticker] = pd.Series(np.log(c[1:]/c[:-1]), index=data[ticker].index[1:])
-    returns_df = pd.DataFrame(all_returns).fillna(0)
-    spy_returns = returns_df.get('SPY', returns_df.iloc[:,0].fillna(0))
-    
-    regime_detector = RegimeDetectorHMM(n_regimes=3)
-    regime_detector.fit(spy_returns)
-    regimes = regime_detector.predict(spy_returns)
-    regime_stats = regime_detector.get_regime_stats(spy_returns)
-    print(f"   Regime distribution:\n{regimes.value_counts().to_string()}")
-    
-    # ---------- 6. RISK ENGINE ----------
-    print("\n[6/12] Computing risk metrics (VaR, CVaR, tail risk)...")
-    risk_engine = RiskEngine()
-    var_metrics = risk_engine.compute_all_var(spy_returns.dropna().values[:1000])
-    tail_risk = risk_engine.compute_tail_risk(spy_returns.dropna().values[:1000])
-    print(f"   VaR 95%: {var_metrics.get('var_95_historical', 0):.4f}")
-    print(f"   CVaR 95%: {var_metrics.get('cvar_95', 0):.4f}")
-    print(f"   Max DD: {tail_risk.get('max_drawdown', 0)*100:.2f}%")
-    
-    # ---------- 7. VOLATILITY + COVARIANCE ----------
-    print("\n[7/12] Building covariance matrix...")
-    vol_engine = VolatilityEngine()
-    for ticker in args.tickers[:5]:
+            close = data[ticker]['Close'].values.flatten()
+            returns_dict[ticker] = pd.Series(np.log(close[1:]/close[:-1]), index=data[ticker].index[1:])
+    returns_df = pd.DataFrame(returns_dict).fillna(0)
+    for ticker in args.tickers:
         if ticker in returns_df.columns:
-            vol_engine.fit_garch(returns_df[ticker].dropna(), ticker)
-    Sigma = vol_engine.build_covariance_matrix(returns_df, returns_df.index[-1])
-    print(f"   Covariance matrix: {Sigma.shape}")
-    
-    # ---------- 8. FACTOR DECOMPOSITION ----------
-    print("\n[8/12] Decomposing returns into style factors...")
-    factor_engine = FactorDecomposition()
-    factor_returns = factor_engine.compute_factor_returns(returns_df.iloc[:500])
-    print(f"   Factors: {list(factor_returns.columns)}")
-    
-    # ---------- 9. PORTFOLIO OPTIMIZATION ----------
-    print("\n[9/12] Running portfolio optimization...")
-    optimizer = PortfolioOptimizer(max_weight=0.25, risk_aversion=2.0, transaction_cost=0.0003)
+            vol_engine.fit_garch(returns_df[ticker], ticker)
     
-    recent_returns = returns_df.iloc[-252:].dropna(axis=1)
-    mu_est = recent_returns.mean().values * 252
-    Sigma_est = recent_returns.cov().values * 252
-    Sigma_est = Sigma_est[:len(mu_est), :len(mu_est)]
+    # Portfolio optimization & backtest
+    print("\n[6/6] Running portfolio backtest...")
+    pred_df = pd.DataFrame({
+        'date': dates[val_end:], 'ticker': tickers_arr[val_end:],
+        'predicted_return': alpha_pred, 'actual_return': y_test
+    })
     
-    max_sharpe = optimizer.optimize_max_sharpe(mu_est, Sigma_est)
-    robust = optimizer.robust_optimization(mu_est, Sigma_est)
+    test_dates = sorted(pd.to_datetime(pred_df['date'].unique()))
+    rebalance_dates = test_dates[::5]
     
-    print(f"   Max Sharpe: {max_sharpe['sharpe_ratio']:.3f} (vol: {max_sharpe['volatility']*100:.1f}%)")
-    print(f"   Robust Sharpe: {robust['sharpe_ratio']:.3f} (vol: {robust['volatility']*100:.1f}%)")
-    
-    # ---------- 10. BACKTEST ----------
-    print("\n[10/12] Running backtest...")
-    backtest_engine = BacktestEngine(initial_capital=args.capital, transaction_cost=0.0003)
+    optimizer = PortfolioOptimizer(max_weight=0.25, risk_aversion=2.0)
+    weights_history = []
     
-    test_dates = pd.to_datetime(pd.Series(dates_test).unique())
-    test_dates = sorted(test_dates)[::5]
+    for rd in rebalance_dates:
+        day_preds = pred_df[pred_df['date'] == rd]
+        if len(day_preds) < 3:
+            continue
+        mu = day_preds.set_index('ticker')['predicted_return'].reindex(args.tickers).fillna(0).values
+        try:
+            Sigma = vol_engine.build_covariance_matrix(returns_df, rd)
+            Sigma = Sigma.reindex(index=args.tickers, columns=args.tickers).fillna(0).values
+        except:
+            Sigma = np.eye(len(args.tickers)) * 0.04
+        result = optimizer.optimize_max_sharpe(mu, Sigma)
+        weights_history.append(pd.Series(result['weights'], index=args.tickers, name=rd))
+    
+    if len(weights_history) == 0:
+        print("No valid rebalance dates. Using equal weights.")
+        print("Backtest cannot proceed without portfolio weights.")
+        return None, None
     
-    weights_history = []
-    for i, date in enumerate(test_dates[:50]):
-        np.random.seed(i)
-        w = np.random.dirichlet(np.ones(len(recent_returns.columns)))
-        weights_history.append(pd.Series(w, index=recent_returns.columns, name=date))
     weights_df = pd.DataFrame(weights_history)
-    
-    bt_returns = returns_df.reindex(columns=recent_returns.columns).reindex(weights_df.index).fillna(0)
-    bt_metrics = backtest_engine.run_backtest(bt_returns, weights_df)
-    
-    # ---------- 11. ADVANCED MODULES ----------
-    print("\n[11/12] Running advanced modules...")
-    
-    # Explainability
-    explainer = ExplainabilityLayer(create_feature_names())
-    importance = explainer.compute_feature_importance(ensemble.xgboost, X_test[:100])
-    
-    # Anomaly Detection
-    anomaly_detector = AnomalyDetector(contamination=0.05)
-    anomaly_features = features_df[[c for c in features_df.columns if c not in ['ticker','close']]].dropna()[:1000]
-    anomaly_detector.fit(anomaly_features)
-    anomalies = anomaly_detector.detect(anomaly_features)
-    
-    # Stress Testing
-    stress_engine = StressTestEngine()
-    portfolio = {col: max_sharpe['weights'][i] for i, col in enumerate(recent_returns.columns[:min(len(recent_returns.columns), len(max_sharpe['weights']))])}
-    stress_results = stress_engine.run_all_scenarios(portfolio, recent_returns.iloc[:100])
-    
-    # Bayesian
-    bayesian = BayesianForecaster()
-    bayesian.update(spy_returns.dropna().values[-252:])
-    bayes_forecast = bayesian.forecast(horizon=5)
-    
-    # Online learning
-    online = OnlineLearner(lookback_window=252)
-    online.partial_fit(X_train[-100:].reshape(-1, X_train.shape[2]), y_train[-100:])
-    drift = online.get_drift_score(X_val[:50].reshape(-1, X_val.shape[2]), y_val[:50])
-    
-    # Hedging
-    hedger = DynamicHedgingEngine(max_hedge_ratio=0.5)
-    hedge_ratio = hedger.compute_hedge_ratio(portfolio_delta=0.3, portfolio_gamma=-0.01, volatility=0.2)
-    
-    # Strategy Ensemble
-    strat_ensemble = StrategyEnsemble()
-    capital_alloc = strat_ensemble.allocate_capital()
-    
-    print(f"   Top feature: {importance.index[0]} ({importance.values[0]:.3f})")
-    print(f"   Anomalies detected: {anomaly_detector.get_anomaly_stats()['n_anomalies']}")
-    print(f"   Concept drift: {drift:.4f}")
-    print(f"   Hedge ratio: {hedge_ratio:.2f}")
-    print(f"   Bayesian prob(positive): {bayes_forecast['prob_positive']:.3f}")
-    
-    # ---------- 12. RESULTS ----------
-    print("\n" + "=" * 70)
-    print("   📊  F I N A L   R E S U L T S")
-    print("=" * 70)
-    
-    final_results = {
-        'alpha_model': {
-            'lstm_val_ic': alpha_metrics['lstm']['val_ic'][-1] if alpha_metrics['lstm']['val_ic'] else 0,
-            'transformer_val_ic': alpha_metrics['transformer']['val_ic'][-1] if alpha_metrics['transformer']['val_ic'] else 0,
-            'xgboost_ic': alpha_metrics['xgboost'].get('ic', 0),
-            'meta_ic': meta_ic['mean_ic']
-        },
-        'backtest': {
-            'sharpe': bt_metrics.get('sharpe_ratio', 0),
-            'sortino': bt_metrics.get('sortino_ratio', 0),
-            'max_drawdown': bt_metrics.get('max_drawdown', 0),
-            'calmar': bt_metrics.get('calmar_ratio', 0),
-            'total_return': bt_metrics.get('total_return', 0),
-            'annualized_return': bt_metrics.get('annualized_return', 0)
-        },
-        'risk': {
-            'var_95': var_metrics.get('var_95_historical', 0),
-            'cvar_95': var_metrics.get('cvar_95', 0),
-            'max_drawdown': tail_risk.get('max_drawdown', 0),
-            'skewness': tail_risk.get('skewness', 0),
-            'kurtosis': tail_risk.get('kurtosis', 0)
-        },
-        'portfolio': {
-            'max_sharpe': max_sharpe['sharpe_ratio'],
-            'robust_sharpe': robust['sharpe_ratio'],
-            'avg_turnover': bt_metrics.get('avg_turnover', 0)
-        },
-        'advanced_modules': {
-            'concept_drift': drift,
-            'hedge_ratio': hedge_ratio,
-            'anomaly_rate': anomaly_detector.get_anomaly_stats().get('anomaly_rate', 0),
-            'bayesian_prob_positive': bayes_forecast['prob_positive'],
-            'strategy_count': len(strat_ensemble.strategies)
-        }
-    }
-    
-    for section, metrics in final_results.items():
-        print(f"\n  [{section}]")
-        for k, v in metrics.items():
-            if isinstance(v, float):
-                print(f"    {k}: {v:.4f}")
-    
-    # Save results
+    backtest_returns = returns_df.reindex(weights_df.index).fillna(0)
+    
+    engine = BacktestEngine(initial_capital=args.capital)
+    bt_results = engine.run_backtest(backtest_returns, weights_df, rebalance_dates=weights_df.index)
+    
+    # Regime detection
+    if 'SPY' in returns_df.columns:
+        regime = RegimeDetector()
+        spy_rets = returns_df['SPY'].reindex(weights_df.index).fillna(0)
+        regimes = regime.detect_regimes(spy_rets)
+        regime_stats = regime.get_regime_stats(spy_rets)
+        print("\nRegime Statistics:")
+        print(regime_stats.to_string())
+    
+    # Print results
+    print("\n" + "=" * 60)
+    print("BACKTEST RESULTS")
+    print("=" * 60)
+    for k, v in bt_results.items():
+        if isinstance(v, float):
+            print(f"{k:>25s}: {v:.4f}")
+        else:
+            print(f"{k:>25s}: {v}")
+    
+    # Save
     os.makedirs(args.output, exist_ok=True)
-    with open(f"{args.output}/final_results.json", 'w') as f:
-        json.dump(final_results, f, indent=2, default=str)
-    
-    stress_results.to_csv(f"{args.output}/stress_tests.csv")
-    regime_stats.to_csv(f"{args.output}/regime_stats.csv")
-    
-    print(f"\n[12/12] ✅ Results saved to {args.output}/")
-    print(f"Dashboard: https://huggingface.co/spaces/Premchan369/alphaforge-dashboard")
+    with open(f"{args.output}/backtest_results.json", 'w') as f:
+        json.dump({k: str(v) for k, v in bt_results.items()}, f, indent=2)
     
-    return final_results
+    return bt_results, engine
 
 
-def parse_args():
-    parser = argparse.ArgumentParser(description='AlphaForge - Autonomous Quant Fund OS')
-    parser.add_argument('--tickers', type=str, nargs='+', default=['SPY','QQQ','AAPL','MSFT','GOOGL','AMZN','META','NVDA','TSLA','JPM'])
-    parser.add_argument('--start', type=str, default='2020-01-01')
-    parser.add_argument('--end', type=str, default='2024-01-01')
-    parser.add_argument('--lookback', type=int, default=60)
-    parser.add_argument('--horizon', type=int, default=5)
-    parser.add_argument('--epochs', type=int, default=30)
-    parser.add_argument('--device', type=str, default='cpu')
-    parser.add_argument('--capital', type=float, default=1_000_000)
-    parser.add_argument('--output', type=str, default='results/')
-    return parser.parse_args()
+def run_realtime(args):
+    """Run real-time streaming pipeline"""
+    print("=" * 60)
+    print("ALPHA FORGE v2.0 - Real-Time Pipeline")
+    print("=" * 60)
+    
+    engine = RealtimeFeatureEngine(
+        tickers=args.tickers,
+        data_source=args.source,
+        api_key=args.api_key,
+        secret_key=args.secret_key,
+        include_sentiment=args.include_sentiment
+    )
+    
+    print(f"\nStarting {args.source} data stream for {len(args.tickers)} tickers...")
+    print(f"Tickers: {', '.join(args.tickers[:5])}{'...' if len(args.tickers) > 5 else ''}")
+    print("\nPress Ctrl+C to stop.\n")
+    
+    engine.start(interval='1m', poll_seconds=60)
+    
+    try:
+        import time
+        while True:
+            time.sleep(10)
+            for t in args.tickers[:3]:
+                df = engine.get_latest(t, lookback=5)
+                if len(df) > 0:
+                    latest = df.iloc[-1]
+                    sentiment = engine.news.get_latest_sentiment(t, hours=1)
+                    flow = engine.order_flow.get_imbalance(t)
+                    print(f"  {t}: ${latest['Close']:.2f} | Vol: {latest['Volume']:,.0f} | OFI: {flow['ofi']:.3f} | Sent: {len(sentiment)} articles")
+    except KeyboardInterrupt:
+        print("\nStopping...")
+        engine.stop()
+        print("Stopped.")
 
 
-if __name__ == '__main__':
+def main():
     args = parse_args()
-    run_full_pipeline(args)
+    
+    if args.mode == 'train':
+        from market_data import MarketDataPipeline
+        pipeline = MarketDataPipeline(args.tickers, args.start, args.end)
+        data = pipeline.fetch_data()
+        if args.advanced_features:
+            features_df = build_advanced_features(data)
+        else:
+            features_df = pipeline.create_feature_matrix()
+        X, y, _, _ = pipeline.create_sequences(features_df, args.lookback, args.horizon)
+        n = len(X)
+        ensemble = AlphaEnsemble(input_size=X.shape[2], seq_len=args.lookback, device=args.device)
+        ensemble.fit(X[:int(n*0.85)], y[:int(n*0.85)], epochs=args.epochs)
+        os.makedirs(args.output, exist_ok=True)
+        torch.save(ensemble.lstm.state_dict(), f"{args.output}/lstm_model.pt")
+        torch.save(ensemble.transformer.state_dict(), f"{args.output}/transformer_model.pt")
+    
+    elif args.mode == 'backtest':
+        run_backtest(args)
+    
+    elif args.mode == 'realtime':
+        run_realtime(args)
+    
+    elif args.mode == 'options':
+        pricer = MLOptionsPricer(device=args.device)
+        train_df = pricer.generate_synthetic_options(50000)
+        val_df = pricer.generate_synthetic_options(10000)
+        X_train = pricer.prepare_features(train_df)
+        y_train = train_df['price'].values
+        X_val = pricer.prepare_features(val_df)
+        y_val = val_df['price'].values
+        pricer.fit(X_train, y_train, X_val, y_val, epochs=100)
+        os.makedirs(args.output, exist_ok=True)
+        torch.save(pricer.model.state_dict(), f"{args.output}/options_model.pt")
+
+
+if __name__ == '__main__':
+    main()