app.py

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout
import gradio as gr
import plotly.graph_objects as go
from datetime import datetime, timedelta
import warnings
import logging
import traceback
import yfinance as yf

# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class PredictiveSystem:
    def __init__(self):
        self.scaler = StandardScaler()
        self.rf_model = None
        self.lstm_model = None
        self.feature_importance = None
        
    def convert_dates(self, df):
        """Convert date columns to datetime"""
        try:
            df = df.copy()
            # Try to convert 'date' column to datetime
            if 'date' in df.columns:
                df['date'] = pd.to_datetime(df['date'], errors='coerce')
                
                # Extract datetime features
                df['month'] = df['date'].dt.month
                df['day'] = df['date'].dt.day
                df['day_of_week'] = df['date'].dt.dayofweek
                df['is_weekend'] = df['date'].dt.dayofweek.isin([5, 6]).astype(int)
                
                # Drop original date column
                df = df.drop('date', axis=1)
                
            return df
        except Exception as e:
            logger.error(f"Error converting dates: {str(e)}")
            raise
    
    def validate_data(self, df):
        """Validate input data structure and contents"""
        try:
            # Check if dataframe is empty
            if df.empty:
                raise ValueError("The uploaded file contains no data")
                
            # Check minimum number of rows
            if len(df) < 30:
                raise ValueError("Dataset must contain at least 30 rows of data")
                
            # Check for minimum number of columns
            if len(df.columns) < 2:
                raise ValueError("Dataset must contain at least 2 columns (features and target)")
                
            # First convert date columns
            df = self.convert_dates(df)
            
            # Now check for remaining non-numeric columns
            non_numeric_cols = df.select_dtypes(exclude=['number']).columns
            if len(non_numeric_cols) > 0:
                raise ValueError(f"Non-numeric columns found after date processing: {', '.join(non_numeric_cols)}. Please ensure all features are numeric.")
                
            return True
            
        except Exception as e:
            logger.error(f"Data validation error: {str(e)}")
            raise
    
    def preprocess_data(self, df):
        """Clean and preprocess the data with error handling"""
        try:
            logger.info("Starting data preprocessing...")
            
            # Convert dates first
            df_processed = self.convert_dates(df)
            
            # Handle missing values
            missing_count = df_processed.isnull().sum().sum()
            if missing_count > 0:
                logger.info(f"Handling {missing_count} missing values")
                df_processed = df_processed.fillna(method='ffill').fillna(method='bfill')
            
            # Remove any remaining non-numeric columns
            numeric_cols = df_processed.select_dtypes(include=[np.number]).columns
            df_processed = df_processed[numeric_cols]
            
            logger.info("Data preprocessing completed successfully")
            return df_processed
            
        except Exception as e:
            logger.error(f"Error in preprocessing data: {str(e)}")
            raise

    def feature_selection(self, X, y):
        """Select important features using Random Forest with error handling"""
        try:
            logger.info("Starting feature selection...")
            
            rf = RandomForestRegressor(n_estimators=100, random_state=42)
            rf.fit(X, y)
            
            self.feature_importance = pd.DataFrame({
                'feature': X.columns,
                'importance': rf.feature_importances_
            }).sort_values('importance', ascending=False)
            
            selected_features = self.feature_importance['feature'].head(
                min(10, len(X.columns))
            )
            
            logger.info(f"Selected {len(selected_features)} features")
            return X[selected_features]
            
        except Exception as e:
            logger.error(f"Error in feature selection: {str(e)}")
            raise
    
    def train_models(self, X, y):
        """Train both Random Forest and LSTM models with error handling"""
        try:
            logger.info("Starting model training...")
            
            # Split data
            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
            
            # Scale data
            X_train_scaled = self.scaler.fit_transform(X_train)
            X_test_scaled = self.scaler.transform(X_test)
            
            # Train Random Forest
            logger.info("Training Random Forest model...")
            self.rf_model = RandomForestRegressor(n_estimators=100, random_state=42)
            self.rf_model.fit(X_train_scaled, y_train)
            
            # Train LSTM
            logger.info("Training LSTM model...")
            X_train_lstm = X_train_scaled.reshape((X_train_scaled.shape[0], 1, X_train_scaled.shape[1]))
            
            self.lstm_model = Sequential([
                LSTM(50, activation='relu', input_shape=(1, X_train_scaled.shape[1]), return_sequences=True),
                Dropout(0.2),
                LSTM(50, activation='relu'),
                Dense(1)
            ])
            
            self.lstm_model.compile(optimizer='adam', loss='mse')
            
            # Use early stopping
            early_stopping = tf.keras.callbacks.EarlyStopping(
                monitor='loss',
                patience=5,
                restore_best_weights=True
            )
            
            self.lstm_model.fit(
                X_train_lstm, 
                y_train, 
                epochs=50, 
                batch_size=32, 
                verbose=0,
                callbacks=[early_stopping]
            )
            
            # Calculate metrics
            rf_pred = self.rf_model.predict(X_test_scaled)
            lstm_pred = self.lstm_model.predict(
                X_test_scaled.reshape((X_test_scaled.shape[0], 1, X_test_scaled.shape[1]))
            )
            
            metrics = {
                'rf_rmse': np.sqrt(mean_squared_error(y_test, rf_pred)),
                'rf_r2': r2_score(y_test, rf_pred),
                'lstm_rmse': np.sqrt(mean_squared_error(y_test, lstm_pred)),
                'lstm_r2': r2_score(y_test, lstm_pred)
            }
            
            logger.info("Model training completed successfully")
            return metrics
            
        except Exception as e:
            logger.error(f"Error in model training: {str(e)}")
            raise
            
    def generate_predictions(self, X):
        """Generate predictions using both models"""
        try:
            X_scaled = self.scaler.transform(X)
            
            rf_pred = self.rf_model.predict(X_scaled)
            lstm_pred = self.lstm_model.predict(
                X_scaled.reshape((X_scaled.shape[0], 1, X_scaled.shape[1]))
            )
            
            # Combine predictions (ensemble)
            final_pred = (rf_pred + lstm_pred.flatten()) / 2
            
            return final_pred
            
        except Exception as e:
            logger.error(f"Error generating predictions: {str(e)}")
            raise

def fetch_real_time_data(ticker):
    """Fetch real-time stock data using yfinance"""
    try:
        stock = yf.Ticker(ticker)
        data = stock.history(period="1d")
        return data
    except Exception as e:
        logger.error(f"Error fetching real-time data for {ticker}: {str(e)}")
        raise

def create_gradio_interface(predictor):
    def process_and_predict(file, ticker):
        try:
            # Read data
            logger.info("Reading uploaded file...")
            df = pd.read_csv(file.name)
            
            # Show initial data info
            logger.info(f"Columns in uploaded file: {', '.join(df.columns)}")
            logger.info(f"Data types: {df.dtypes}")
            
            # Validate and process data
            predictor.validate_data(df)
            df_processed = predictor.preprocess_data(df)
            
            # Separate features and target
            y = df_processed.iloc[:, -1]  # Assume last column is target
            X = df_processed.iloc[:, :-1]
            
            # Feature selection and model training
            X_selected = predictor.feature_selection(X, y)
            metrics = predictor.train_models(X_selected, y)
            
            # Generate predictions
            predictions = predictor.generate_predictions(X_selected)
            
            # Fetch real-time stock data
            real_time_data = fetch_real_time_data(ticker)
            
            # Create visualization
            fig = go.Figure()
            fig.add_trace(go.Scatter(y=y, name='Actual', line=dict(color='blue')))
            fig.add_trace(go.Scatter(y=predictions, name='Predicted', line=dict(color='red')))
            fig.add_trace(go.Scatter(y=real_time_data['Close'], name='Real-Time Data', line=dict(color='green')))
            fig.update_layout(
                title='Actual vs Predicted vs Real-Time Values',
                xaxis_title='Time',
                yaxis_title='Value',
                template='plotly_white'
            )
            
            # Format output
            output = f"""
            Model Performance Metrics:
            Random Forest RMSE: {metrics['rf_rmse']:.4f}
            Random Forest R²: {metrics['rf_r2']:.4f}
            LSTM RMSE: {metrics['lstm_rmse']:.4f}
            LSTM R²: {metrics['lstm_r2']:.4f}
            
            Data Processing Summary:
            - Total records processed: {len(df)}
            - Features selected: {len(X_selected.columns)}
            - Date features created: month, day, day_of_week, is_weekend
            - Training completed successfully
            
            Real-Time Data Summary:
            - Ticker: {ticker}
            - Last Close Price: {real_time_data['Close'].iloc[-1]:.2f}
            """
            
            logger.info("Analysis completed successfully")
            return fig, output
            
        except Exception as e:
            error_msg = f"""
            Error occurred during processing:
            {str(e)}
            
            Please ensure your data:
            1. Is in CSV format
            2. Contains a 'date' column (will be automatically processed)
            3. Contains numeric feature columns
            4. Has at least 30 rows of data
            5. Has both feature columns and a target column
            6. Has no corrupted values
            
            Technical details for debugging:
            {traceback.format_exc()}
            """
            logger.error(f"Process failed: {str(e)}")
            return None, error_msg
    
    interface = gr.Interface(
        fn=process_and_predict,
        inputs=[
            gr.File(label="Upload CSV file"),
            gr.Textbox(label="Stock Ticker (e.g., AAPL)")
        ],
        outputs=[
            gr.Plot(label="Predictions Visualization"),
            gr.Textbox(label="Analysis Results", lines=10)
        ],
        title="Predictive & Prescriptive Analytics System",
        description="""
        Upload your CSV file containing historical data and enter a stock ticker to fetch real-time data.
        Required format: Furtur Any contact Anupam Joshi 91-9878255748 @ joshianupam32@gmail.com
        - A 'date' column in any standard date format
        - Numeric feature columns
        - A target column (last column)
        - At least 30 rows of data
        
        The system will automatically:
        - Process the date column into useful features
        - Handle any missing values
        - Select the most important features
        - Train and evaluate the models
        - Fetch and display real-time stock data
        """,
        examples=[["sample_sales_data.csv", "AAPL"]]
    )
    
    return interface

# Initialize and launch
if __name__ == "__main__":
    try:
        predictor = PredictiveSystem()
        interface = create_gradio_interface(predictor)
        interface.launch(share=True)
    except Exception as e:
        logger.error(f"Failed to launch interface: {str(e)}")
        raise