app.py

import gradio as gr
import pandas as pd
import numpy as np
import plotly.graph_objects as go
import os
import warnings
import datetime
import traceback
import shutil
import tempfile

# Disable Gradio queueing system (FIXES KeyError: 1 errors)
gr.queue = False

# Suppress warnings for cleaner output
warnings.filterwarnings('ignore')

# Performance optimization
os.environ["OMP_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"

# Define data directories
RAW_DATA_DIR = "data/raw"
PROCESSED_DATA_DIR = "data/processed"
os.makedirs(PROCESSED_DATA_DIR, exist_ok=True)

# Predefined trading pairs
TRADING_PAIRS = {
    "EURUSD": {
        "description": "Euro to US Dollar Forex Pair",
        "date_format": "%d.%m.%Y %H:%M:%S.%f %z",
        "has_timezone": True,
        "decimal_separator": ".",
        "required_columns": ["Open", "High", "Low", "Close"]
    }
}

def preprocess_data_file(raw_file_path, pair_name):
    """Preprocess raw data file to standardized format"""
    print(f"🔄 Preprocessing data for {pair_name}...")
    
    try:
        # Read raw data
        df = pd.read_csv(raw_file_path, encoding='utf-8')
        print(f"✅ Successfully read {pair_name} data with utf-8 encoding")
        
        # Standardize column names
        column_mapping = {}
        for col in df.columns:
            col_lower = col.lower().strip()
            if any(keyword in col_lower for keyword in ['date', 'time', 'timestamp']):
                column_mapping[col] = 'datetime'
            elif 'open' in col_lower:
                column_mapping[col] = 'Open'
            elif 'high' in col_lower:
                column_mapping[col] = 'High'
            elif 'low' in col_lower:
                column_mapping[col] = 'Low'
            elif 'close' in col_lower:
                column_mapping[col] = 'Close'
            elif 'volume' in col_lower:
                column_mapping[col] = 'Volume'
        
        if column_mapping:
            df.rename(columns=column_mapping, inplace=True)
            print(f"🏷️ Standardized columns: {list(column_mapping.keys())} → {list(column_mapping.values())}")
        
        # Process datetime column
        datetime_col = None
        for col in ['datetime', 'date', 'time', 'timestamp']:
            if col in df.columns:
                datetime_col = col
                break
        
        if datetime_col is None:
            raise Exception("❌ No datetime column found in data")
        
        # Handle EURUSD special format
        if pair_name == "EURUSD" and df[datetime_col].astype(str).str.contains('GMT').any():
            print("🕗 Handling EURUSD special datetime format...")
            df[datetime_col] = df[datetime_col].str.replace(' GMT', '', regex=False)
            df[datetime_col] = pd.to_datetime(
                df[datetime_col], 
                format="%d.%m.%Y %H:%M:%S.%f %z",
                errors='coerce',
                utc=True
            )
        else:
            df[datetime_col] = pd.to_datetime(
                df[datetime_col], 
                errors='coerce',
                utc=True
            )
        
        # Clean data
        before_count = len(df)
        df = df.dropna(subset=[datetime_col])
        print(f"🧹 Removed {before_count - len(df)} rows with invalid dates")
        
        # Set datetime as index
        df.set_index(datetime_col, inplace=True)
        df.sort_index(inplace=True)
        
        # Fill missing values
        for col in ['Open', 'High', 'Low', 'Close']:
            if col in df.columns:
                missing_before = df[col].isna().sum()
                if missing_before > 0:
                    df[col] = df[col].fillna(method='ffill').fillna(method='bfill')
                    print(f"  🔄 Filled {missing_before} missing values in {col}")
        
        # Remove duplicates
        before_count = len(df)
        df = df[~df.index.duplicated(keep='first')]
        print(f"🧹 Removed {before_count - len(df)} duplicate entries")
        
        # Save preprocessed data
        processed_file = os.path.join(PROCESSED_DATA_DIR, f"{pair_name}_processed.csv")
        df.to_csv(processed_file)
        print(f"✅ Saved preprocessed data to {processed_file}")
        
        return df
        
    except Exception as e:
        print(f"❌ Preprocessing error for {pair_name}: {str(e)}")
        traceback.print_exc()
        return None

def load_available_data():
    """Load and preprocess all available data files"""
    global available_data
    available_data = {}
    
    # Check if raw data directory exists
    if not os.path.exists(RAW_DATA_DIR):
        print(f"⚠️ Raw data directory not found: {RAW_DATA_DIR}")
        # Check if data is in root directory instead
        if os.path.exists("data") and os.path.isdir("data"):
            for filename in os.listdir("data"):
                if filename.endswith('.csv'):
                    os.makedirs(RAW_DATA_DIR, exist_ok=True)
                    shutil.move(os.path.join("data", filename), os.path.join(RAW_DATA_DIR, filename))
                    print(f"✅ Moved {filename} to {RAW_DATA_DIR}")
    
    if not os.path.exists(RAW_DATA_DIR):
        print(f"❌ Still cannot find raw data directory: {RAW_DATA_DIR}")
        return available_data
    
    print(f"🔍 Scanning for data files in {RAW_DATA_DIR}...")
    
    for filename in os.listdir(RAW_DATA_DIR):
        if filename.endswith('.csv'):
            pair_name = filename.split('.')[0].upper()
            
            if pair_name not in TRADING_PAIRS:
                TRADING_PAIRS[pair_name] = {
                    "description": f"{pair_name} Trading Pair",
                    "date_format": "%Y-%m-%d %H:%M:%S",
                    "has_timezone": False,
                    "decimal_separator": ".",
                    "required_columns": ["Open", "High", "Low", "Close"]
                }
            
            raw_file_path = os.path.join(RAW_DATA_DIR, filename)
            processed_file_path = os.path.join(PROCESSED_DATA_DIR, f"{pair_name}_processed.csv")
            
            # Check for existing preprocessed file
            if os.path.exists(processed_file_path):
                try:
                    df = pd.read_csv(processed_file_path, index_col=0, parse_dates=True)
                    available_data[pair_name] = df
                    print(f"✅ Using existing preprocessed data for {pair_name} with {len(df)} records")
                    continue
                except Exception as e:
                    print(f"⚠️ Error loading preprocessed file: {str(e)}. Reprocessing.")
            
            # Preprocess the file
            print(f"🔄 Processing {pair_name} data...")
            df = preprocess_data_file(raw_file_path, pair_name)
            if df is not None:
                available_data[pair_name] = df
                print(f"✅ Successfully loaded {pair_name} with {len(df)} records")
    
    return available_data

def get_available_pairs():
    """Get list of available trading pairs with status"""
    if not available_data:
        return "⚠️ No data files found. Please upload CSV files to the 'data/raw' directory."
    
    status = "✅ Available trading pairs:\n"
    for pair in sorted(available_data.keys()):
        df = available_data[pair]
        records = len(df)
        if records > 0:
            date_range = f"{df.index.min().strftime('%Y-%m-%d')} to {df.index.max().strftime('%Y-%m-%d')}"
            status += f"• {pair}: {records} records ({date_range})\n"
        else:
            status += f"• {pair}: 0 records (Data Error)\n"
    return status

def analyze_trading_pair(pair_name: str):
    """Analyze a specific trading pair"""
    pair_name = pair_name.upper().strip()
    print(f"\n🔍 Starting analysis for {pair_name}")
    
    # Error fallbacks
    default_error_fig = go.Figure().update_layout(
        title="Analysis Failed", 
        xaxis_title="Date", 
        yaxis_title="Price",
        template="plotly_white",
        height=500
    )
    default_error_df = gr.DataFrame(
        headers=["Error"], 
        value=[["Analysis failed - check logs for details"]],
        interactive=False
    )
    
    # Check if data is available
    if pair_name not in available_data:
        available_pairs = ", ".join(available_data.keys()) or "None"
        return (
            f"❌ Data not available for '{pair_name}'\nAvailable pairs: {available_pairs}",
            default_error_fig, 
            default_error_fig, 
            default_error_df
        )
    
    try:
        hist = available_data[pair_name].copy()
        
        # Basic data validation
        required_cols = ['Open', 'High', 'Low', 'Close']
        if not all(col in hist.columns for col in required_cols):
            missing_cols = [col for col in required_cols if col not in hist.columns]
            return (
                f"❌ Missing required columns: {', '.join(missing_cols)}\nAvailable columns: {', '.join(hist.columns)}",
                default_error_fig, 
                default_error_fig, 
                default_error_df
            )
        
        # --- 1. Candlestick Chart with Technical Indicators (MAs) ---
        fig = go.Figure()
        
        # Add candlestick
        fig.add_trace(go.Candlestick(
            x=hist.index,
            open=hist['Open'],
            high=hist['High'],
            low=hist['Low'],
            close=hist['Close'],
            name='Price'
        ))
        
        # Add moving averages
        if len(hist) >= 20:
            hist['MA20'] = hist['Close'].rolling(window=20, min_periods=1).mean()
            fig.add_trace(go.Scatter(
                x=hist.index,
                y=hist['MA20'],
                mode='lines',
                name='20-period MA',
                line=dict(color='blue', width=1.5)
            ))
        
        if len(hist) >= 50:
            hist['MA50'] = hist['Close'].rolling(window=50, min_periods=1).mean()
            fig.add_trace(go.Scatter(
                x=hist.index,
                y=hist['MA50'],
                mode='lines',
                name='50-period MA',
                line=dict(color='orange', width=1.5)
            ))
        
        fig.update_layout(
            title=f"{pair_name} Price Analysis",
            xaxis_title="Date",
            yaxis_title="Price",
            template="plotly_white",
            hovermode="x unified",
            height=500,
            margin=dict(l=50, r=50, t=50, b=50)
        )
        
        # --- 2. Simple Forecast (without Prophet to avoid import issues) ---
        forecast_fig = default_error_fig
        forecast_table = default_error_df
        forecast_result = "Forecast functionality will be available soon."
        
        try:
            # Simple linear forecast as fallback
            if len(hist) >= 30:
                # Take last 30 days
                recent_data = hist['Close'].tail(30)
                dates = recent_data.index
                
                # Create simple trend line
                x = np.arange(len(recent_data))
                y = recent_data.values
                slope, intercept = np.polyfit(x, y, 1)
                
                # Create forecast data
                future_dates = [dates[-1] + datetime.timedelta(days=i) for i in range(1, 31)]
                future_values = [slope * (len(x) + i) + intercept for i in range(30)]
                
                # Create forecast chart
                forecast_fig = go.Figure()
                forecast_fig.add_trace(go.Scatter(
                    x=dates,
                    y=recent_data.values,
                    mode='lines',
                    name='Historical',
                    line=dict(color='blue', width=2)
                ))
                forecast_fig.add_trace(go.Scatter(
                    x=future_dates,
                    y=future_values,
                    mode='lines',
                    name='Forecast',
                    line=dict(color='red', width=2, dash='dash')
                ))
                forecast_fig.update_layout(
                    title=f"{pair_name} 30-Day Price Forecast (Simple Trend)",
                    xaxis_title="Date",
                    yaxis_title="Price",
                    template="plotly_white",
                    height=500,
                    hovermode="x unified"
                )
                
                # Create forecast table
                table_data = []
                for i, (date, value) in enumerate(zip(future_dates, future_values)):
                    trend = "📈 Rising" if slope > 0 else "📉 Falling"
                    table_data.append([
                        date.strftime('%Y-%m-%d'),
                        f"{value:.5f}",
                        f"{value * 0.98:.5f}",
                        f"{value * 1.02:.5f}",
                        trend
                    ])
                
                forecast_table = gr.DataFrame(
                    headers=["Date", "Predicted Price", "Lower Bound", "Upper Bound", "Trend"],
                    value=table_data,
                    datatype=["str", "str", "str", "str", "str"],
                    label=f"{pair_name} 30-Day Price Forecast Table",
                    interactive=False
                )
                
                forecast_result = (
                    f"🔮 30-Day Forecast (Simple Trend):\n"
                    f"Projected price range based on recent trend"
                )
        
        except Exception as e:
            print(f"⚠️ Forecasting error: {str(e)}")
            forecast_result = f"⚠️ Forecasting error: {str(e)}"
        
        # Technical analysis
        current_price = hist['Close'].iloc[-1]
        signal = "📊 Analyzing market conditions..."
        
        if 'MA20' in hist.columns and 'MA50' in hist.columns:
            ma20 = hist['MA20'].iloc[-1]
            ma50 = hist['MA50'].iloc[-1]
            
            if current_price > ma20 > ma50:
                signal = "🚀 STRONG BULLISH: Golden Cross pattern"
            elif current_price < ma20 < ma50:
                signal = "💣 STRONG BEARISH: Death Cross pattern"
            elif current_price > ma20:
                signal = "📈 BULLISH: Price above 20-period MA"
            else:
                signal = "📉 BEARISH: Price below 20-period MA"
        
        # Calculate performance metrics
        start_price = hist['Close'].iloc[0]
        total_return = (current_price / start_price - 1) * 100
        volatility = hist['Close'].pct_change().std() * np.sqrt(252) * 100
        
        # Create result text
        result_text = (
            f"📊 {pair_name} Analysis Report\n"
            f"{'=' * 40}\n"
            f"💰 Current Price: {current_price:.5f}\n"
            f"📈 Total Return: {total_return:.2f}%\n"
            f"⚡ Volatility: {volatility:.2f}%\n"
            f"🎯 Signal: {signal}\n"
            f"{'=' * 40}\n"
            f"{forecast_result}"
        )
        
        print(f"✅ Analysis completed for {pair_name}")
        return result_text, fig, forecast_fig, forecast_table
    
    except Exception as e:
        error_msg = f"❌ Analysis error: {str(e)}"
        print(error_msg)
        traceback.print_exc()
        return error_msg, default_error_fig, default_error_fig, default_error_df

def export_forecast(pair_name):
    """Export forecast data to CSV file"""
    try:
        # Create a simple export file
        temp_dir = tempfile.mkdtemp()
        export_path = os.path.join(temp_dir, f"{pair_name}_forecast.csv")
        
        # Create dummy data for now
        dates = [datetime.datetime.now() + datetime.timedelta(days=i) for i in range(30)]
        prices = [1.0800 + i*0.0005 for i in range(30)]
        
        pd.DataFrame({
            'Date': [d.strftime('%Y-%m-%d') for d in dates],
            'Predicted_Price': prices,
            'Lower_Bound': [p * 0.998 for p in prices],
            'Upper_Bound': [p * 1.002 for p in prices],
            'Trend': ['Rising' if prices[i] > prices[i-1] else 'Falling' for i in range(30)]
        }).to_csv(export_path, index=False)
        
        return export_path
    except Exception as e:
        print(f"❌ Export error: {str(e)}")
        return None

def refresh_data():
    """Refresh available data"""
    global available_data
    print("🔄 Refreshing data...")
    available_data = load_available_data()
    return get_available_pairs(), f"📈 Trading Analysis System v2.4\n🕒 Last updated: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}\n🧮 Loaded pairs: {len(available_data)}"

# Load available data at startup
print("🚀 Initializing data processing system...")
available_data = load_available_data()
print(f"📊 Available trading pairs: {list(available_data.keys())}")

# Create Gradio interface
with gr.Blocks(title="Trading Pair AI Analyzer") as demo:
    gr.Markdown("# 📈 Trading Pair AI Analysis System")
    gr.Markdown("### Analyze forex data with interactive charts and forecasts")
    
    with gr.Row():
        with gr.Column(scale=2):
            data_status = gr.Textbox(
                label="📊 Available Data", 
                value=get_available_pairs(),
                interactive=False,
                lines=5
            )
        
        with gr.Column(scale=1):
            gr.Markdown("### ℹ️ System Information")
            system_info = gr.Textbox(
                value=f"📈 Trading Analysis System v2.4\n🕒 Last updated: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}\n🧮 Loaded pairs: {len(available_data)}",
                interactive=False,
                lines=3
            )
    
    refresh_btn = gr.Button("🔄 Refresh Data", variant="secondary")
    
    with gr.Row():
        with gr.Column(scale=2):
            pair_input = gr.Textbox(
                label="🔍 Trading Pair to Analyze", 
                value=list(available_data.keys())[0] if available_data else "EURUSD",
                placeholder="Enter pair name (e.g., EURUSD)"
            )
            analyze_btn = gr.Button("🚀 Analyze Pair", variant="primary")
        
        with gr.Column(scale=1):
            export_btn = gr.Button("📥 Export Forecast Data", variant="secondary")
            export_output = gr.File(label="Download Forecast CSV", visible=False)
    
    result_output = gr.Textbox(label="📝 Analysis Results", lines=8)
    
    with gr.Tabs():
        with gr.TabItem("📈 Price Chart & Indicators"):
            price_chart = gr.Plot(label="Candlestick Chart with Moving Averages")
        
        with gr.TabItem("🔮 Price Forecast Chart"):
            forecast_chart = gr.Plot(label="30-Day Price Forecast")
        
        with gr.TabItem("📋 Forecast Table"):
            forecast_table = gr.DataFrame(
                headers=["Date", "Predicted Price", "Lower Bound", "Upper Bound", "Trend"],
                value=[],
                datatype=["str", "str", "str", "str", "str"],
                label="30-Day Price Forecast Table",
                interactive=False
            )
    
    with gr.Accordion("📁 Data Upload Instructions", open=False):
        gr.Markdown("""
        ### How to Add Your Own Data
        
        1. **Prepare your CSV file** with these columns:
           - Date/Time column (any format)
           - Open, High, Low, Close prices
           - Volume (optional)
        
        2. **Upload to Hugging Face Space**:
           - Go to your Space Files tab
           - Create directories: `data/raw/`
           - Upload your CSV files to `data/raw/`
           - Example filenames: `EURUSD.csv`
        
        3. **Refresh the application**:
           - Click the "🔄 Refresh Data" button
           - Wait for data to load
        
        4. **Your data will be automatically preprocessed** and ready for analysis!
        """)
    
    # Event handlers
    analyze_btn.click(
        fn=analyze_trading_pair,
        inputs=pair_input,
        outputs=[result_output, price_chart, forecast_chart, forecast_table]
    )
    
    refresh_btn.click(
        fn=refresh_data,
        inputs=[],
        outputs=[data_status, system_info]
    )
    
    export_btn.click(
        fn=export_forecast,
        inputs=pair_input,
        outputs=export_output
    ).then(
        fn=lambda: gr.update(visible=True),
        inputs=None,
        outputs=export_output
    )

# Launch the app
if __name__ == "__main__":
    demo.launch(
        server_name="0.0.0.0",
        server_port=7860,
        share=False
    )