Update app.py

app.py

@@ -1,359 +1,138 @@
 import gradio as gr
-from smolagents import HfApiModel, CodeAgent
-from huggingface_hub import login
-import os
-import shutil
-import wandb
-import time
-import psutil
-import optuna
-import ast
 import pandas as pd
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.metrics import (accuracy_score, precision_score, 
-                           recall_score, f1_score, classification_report)
-from sklearn.preprocessing import LabelEncoder
 import shap
-import lime
 import lime.lime_tabular
+import wandb
+import optuna
 import matplotlib.pyplot as plt
 import seaborn as sns
-from optuna.visualization import (plot_optimization_history, 
-                                 plot_param_importances,
-                                 plot_parallel_coordinate)
-from PIL import Image
-import base64
-from io import BytesIO
+import tempfile
+import os
 
-# Authenticate Hugging Face
-hf_token = os.getenv("HF_TOKEN")
-login(token=hf_token, add_to_git_credential=True)
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import train_test_split, cross_val_score
+from sklearn.metrics import classification_report, accuracy_score, precision_score, recall_score, f1_score
 
-# Initialize Model
-model = HfApiModel("mistralai/Mixtral-8x7B-Instruct-v0.1", token=hf_token)
+from huggingface_hub import login
+from smolagents import HfApiModel, CodeAgent
 
-# Initialize W&B
-wandb.login(key=os.environ.get('WANDB_API_KEY'))
+# Authenticate with Hugging Face using environment token
+login(os.getenv("HUGGINGFACEHUB_API_TOKEN"))
 
-def save_figure(fig, filename):
-    """Helper function to save matplotlib figures"""
-    os.makedirs('./figures', exist_ok=True)
-    path = f"./figures/{filename}"
-    fig.savefig(path, bbox_inches='tight')
-    plt.close(fig)
-    return path
+# Initialize LLM model and CodeAgent
+llm_model = HfApiModel("meta-llama/Llama-3.1-70B-Instruct")
+agent = CodeAgent(
+    tools=[],
+    model=llm_model,
+    additional_authorized_imports=["numpy", "pandas", "matplotlib.pyplot", "seaborn"],
+    max_iterations=10,
+)
 
-def encode_categorical_data(df):
-    """Encode categorical columns and return encoded df and encoders"""
-    encoders = {}
-    for col in df.select_dtypes(include=['object', 'category']).columns:
-        le = LabelEncoder()
-        df[col] = le.fit_transform(df[col].astype(str))
-        encoders[col] = le
-    return df, encoders
+# Global DataFrame
+df_global = None
 
-def generate_data_insights(df):
-    """Generate insights using smolagent"""
-    agent = CodeAgent(
-        tools=[], 
-        model=model,
-        additional_authorized_imports=["numpy", "pandas", "matplotlib.pyplot", "seaborn"]
-    )
-    
-    prompt = """
-    Analyze this dataset and provide:
-    1. 5 key statistical insights about the data
-    2. 5 suggested visualizations with explanations
-    3. Data quality assessment
-    4. Recommendations for preprocessing
-    
-    For each insight:
-    - Explain its significance
-    - Provide the Python code to verify it
-    - Suggest potential actions
-    
-    Return the results as a dictionary with:
-    - 'insights': List of 5 key insights
-    - 'visualizations': List of 5 visualization descriptions with code
-    - 'quality': Data quality assessment
-    - 'recommendations': Preprocessing recommendations
-    """
-    
-    return agent.run(prompt, additional_args={"df": df})
+# Load and clean data
+def load_data(file):
+    global df_global
+    ext = os.path.splitext(file.name)[-1]
+    if ext in [".csv"]:
+        df = pd.read_csv(file.name)
+    else:
+        df = pd.read_excel(file.name)
+    df = df.dropna(how='all', axis=1).dropna(how='all', axis=0)
+    df = df.fillna(df.mean(numeric_only=True))
+    df_global = df
+    return df.head()
 
-def create_visualizations(df, insights):
-    """Create visualizations based on insights"""
-    visuals = []
+# Use SmolAgent to generate insights and visuals
+def get_insights(_):
+    if df_global is None:
+        return "No data loaded yet."
     try:
-        # Visualization 1: Missing values heatmap
-        if df.isnull().any().any():
-            fig, ax = plt.subplots(figsize=(10, 6))
-            sns.heatmap(df.isnull(), cbar=False, ax=ax)
-            plt.title("Missing Values Heatmap")
-            visuals.append(save_figure(fig, "missing_values.png"))
-        
-        # Visualization 2: Correlation heatmap
-        numeric_cols = df.select_dtypes(include=np.number).columns
-        if len(numeric_cols) > 1:
-            fig, ax = plt.subplots(figsize=(10, 8))
-            sns.heatmap(df[numeric_cols].corr(), annot=True, cmap='coolwarm', ax=ax)
-            plt.title("Correlation Heatmap")
-            visuals.append(save_figure(fig, "correlation_heatmap.png"))
-        
-        # Visualization 3: Feature distributions
-        for col in numeric_cols[:3]:  # First 3 numeric columns
-            fig, ax = plt.subplots(figsize=(10, 6))
-            sns.histplot(df[col], kde=True, ax=ax)
-            plt.title(f"Distribution of {col}")
-            visuals.append(save_figure(fig, f"distribution_{col}.png"))
-        
-        # Visualization 4: Pairplot (sample if large)
-        if len(numeric_cols) > 1:
-            fig = sns.pairplot(df[numeric_cols].sample(min(100, len(df))))
-            visuals.append(save_figure(fig, "pairplot.png"))
-        
-        # Visualization 5: Categorical counts
-        cat_cols = df.select_dtypes(include=['object', 'category']).columns
-        for col in cat_cols[:2]:  # First 2 categorical columns
-            fig, ax = plt.subplots(figsize=(10, 6))
-            df[col].value_counts().plot(kind='bar', ax=ax)
-            plt.title(f"Count of {col}")
-            visuals.append(save_figure(fig, f"count_{col}.png"))
-            
+        result = agent.run(df_global, instructions="Generate 5 data insights and 5 data visualizations.")
+        return result
     except Exception as e:
-        print(f"Visualization error: {e}")
-    
-    return visuals
+        return f"Error from SmolAgent: {e}"
 
-def analyze_data(csv_file, additional_notes=""):
-    """Main data analysis function"""
-    start_time = time.time()
-    
-    # Initialize W&B run
-    run = wandb.init(project="data-analysis", config={
-        "model": "Mixtral-8x7B",
-        "notes": additional_notes,
-        "file": csv_file.name if csv_file else None
-    })
-    
-    try:
-        # Load data
-        df = pd.read_csv(csv_file)
-        
-        # Generate insights with smolagent
-        insights = generate_data_insights(df)
-        
-        # Create visualizations
-        visuals = create_visualizations(df, insights)
-        
-        # Log to W&B
-        for viz in visuals:
-            wandb.log({"visualizations": wandb.Image(viz)})
-        
-        # Format report
-        report = format_analysis_report(insights, visuals)
-        
-        # Track performance
-        execution_time = time.time() - start_time
-        wandb.log({"execution_time": execution_time})
-        
-        return report, visuals
-        
-    except Exception as e:
-        return f"Error: {str(e)}", []
-    finally:
-        run.finish()
+# Train model + hyperparameter tuning
+def run_model(_):
+    wandb_run = wandb.init(project="huggingface_smol_data_analysis", name="Optuna_Tuning", reinit=True)
+    target = df_global.columns[-1]
+    X = df_global.drop(target, axis=1)
+    y = df_global[target]
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
 
-def objective(trial, X, y):
-    """Optuna objective function for hyperparameter tuning"""
-    params = {
-        'n_estimators': trial.suggest_int('n_estimators', 50, 500),
-        'max_depth': trial.suggest_int('max_depth', 3, 15),
-        'min_samples_split': trial.suggest_int('min_samples_split', 2, 10),
-        'min_samples_leaf': trial.suggest_int('min_samples_leaf', 1, 5),
-        'max_features': trial.suggest_categorical('max_features', ['sqrt', 'log2', None]),
-        'bootstrap': trial.suggest_categorical('bootstrap', [True, False]),
-        'criterion': trial.suggest_categorical('criterion', ['gini', 'entropy'])
-    }
-    
-    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
-    
-    model = RandomForestClassifier(**params, random_state=42, n_jobs=-1)
-    model.fit(X_train, y_train)
-    
-    y_pred = model.predict(X_test)
-    
-    # Track multiple metrics
-    metrics = {
-        'accuracy': accuracy_score(y_test, y_pred),
-        'precision': precision_score(y_test, y_pred, average='weighted'),
-        'recall': recall_score(y_test, y_pred, average='weighted'),
-        'f1': f1_score(y_test, y_pred, average='weighted')
+    def objective(trial):
+        n_estimators = trial.suggest_int("n_estimators", 10, 200)
+        max_depth = trial.suggest_int("max_depth", 2, 32, log=True)
+        clf = RandomForestClassifier(n_estimators=n_estimators, max_depth=max_depth)
+        score = cross_val_score(clf, X_train, y_train, n_jobs=-1, cv=3).mean()
+        wandb.log({"cv_score": score, "n_estimators": n_estimators, "max_depth": max_depth})
+        return score
+
+    study = optuna.create_study(direction="maximize")
+    study.optimize(objective, n_trials=20)
+
+    best_params = study.best_params
+    best_model = RandomForestClassifier(**best_params)
+    best_model.fit(X_train, y_train)
+    y_pred = best_model.predict(X_test)
+
+    scores = {
+        "accuracy": accuracy_score(y_test, y_pred),
+        "precision": precision_score(y_test, y_pred, average="weighted", zero_division=0),
+        "recall": recall_score(y_test, y_pred, average="weighted", zero_division=0),
+        "f1_score": f1_score(y_test, y_pred, average="weighted", zero_division=0)
     }
-    
-    # Log to W&B
-    wandb.log({**params, **metrics})
-    
-    return metrics['accuracy']
 
-def tune_hyperparameters(csv_file, n_trials=50):
-    """Hyperparameter tuning with Optuna and W&B"""
-    run = wandb.init(project="hyperparameter-tuning", config={
-        "n_trials": n_trials,
-        "model": "RandomForest"
-    })
-    
-    try:
-        # Load and prepare data
-        df = pd.read_csv(csv_file)
-        df, _ = encode_categorical_data(df)
-        
-        y = df.iloc[:, -1]  # Assume last column is target
-        X = df.iloc[:, :-1]
-        
-        # Optuna study
-        study = optuna.create_study(
-            direction='maximize',
-            sampler=optuna.samplers.TPESampler(),
-            pruner=optuna.pruners.MedianPruner()
-        )
-        
-        study.optimize(lambda trial: objective(trial, X, y), n_trials=n_trials)
-        
-        # Generate visualizations
-        visuals = []
-        fig = plot_optimization_history(study)
-        visuals.append(save_figure(fig, "optimization_history.png"))
-        
-        fig = plot_param_importances(study)
-        visuals.append(save_figure(fig, "param_importance.png"))
-        
-        fig = plot_parallel_coordinate(study)
-        visuals.append(save_figure(fig, "parallel_coordinate.png"))
-        
-        # Train best model
-        best_model = RandomForestClassifier(**study.best_params, random_state=42)
-        best_model.fit(X, y)
-        
-        # SHAP explainability
-        explainer = shap.TreeExplainer(best_model)
-        shap_values = explainer.shap_values(X)
-        
-        fig, ax = plt.subplots(figsize=(10, 8))
-        shap.summary_plot(shap_values, X, show=False)
-        visuals.append(save_figure(fig, "shap_summary.png"))
-        
-        # LIME explainability
-        explainer = lime.lime_tabular.LimeTabularExplainer(
-            X.values,
-            feature_names=X.columns,
-            class_names=['class_0', 'class_1'],  # Modify as needed
-            mode='classification'
-        )
-        exp = explainer.explain_instance(X.iloc[0].values, best_model.predict_proba)
-        fig = exp.as_pyplot_figure()
-        visuals.append(save_figure(fig, "lime_explanation.png"))
-        
-        # Format results
-        report = format_tuning_results(study, best_model, X, y)
-        
-        return report, visuals
-        
-    except Exception as e:
-        return f"Error: {str(e)}", []
-    finally:
-        run.finish()
+    wandb.log(scores)
+    wandb_run.finish()
+
+    top_params_report = pd.DataFrame(study.trials_dataframe().sort_values(by="value", ascending=False).head(7))
+
+    return scores, top_params_report
+
+# SHAP + LIME Explainability
+def explainability(_):
+    target = df_global.columns[-1]
+    X = df_global.drop(target, axis=1)
+    y = df_global[target]
+
+    model = RandomForestClassifier()
+    model.fit(X, y)
+
+    explainer = shap.Explainer(model, X)
+    shap_values = explainer(X)
+    shap.plots.beeswarm(shap_values, show=False)
+    plt.tight_layout()
+    shap_file = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
+    plt.savefig(shap_file.name)
+    plt.close()
+
+    lime_explainer = lime.lime_tabular.LimeTabularExplainer(X.values, feature_names=X.columns, class_names=list(set(y)), discretize_continuous=True)
+    exp = lime_explainer.explain_instance(X.iloc[0].values, model.predict_proba)
+    lime_html = exp.as_html()
+
+    wandb.log({"shap": wandb.Image(shap_file.name), "lime": lime_html})
+
+    return shap_file.name, lime_html
 
-def format_analysis_report(insights, visuals):
-    """Format the analysis report with insights and visuals"""
-    report = f"""
-    <div style="font-family: Arial; max-width: 1000px; margin: 0 auto;">
-        <h1 style="color: #2B547E;">📊 Data Analysis Report</h1>
-        
-        <div style="margin-top: 20px; background: #f8f9fa; padding: 20px; border-radius: 8px;">
-            <h2 style="color: #2B547E;">🔍 Key Insights</h2>
-            {format_insights_section(insights.get('insights', []))}
-        </div>
-        
-        <div style="margin-top: 30px;">
-            <h2 style="color: #2B547E;">📈 Visualizations</h2>
-            {format_visualizations(visuals)}
-        </div>
-    </div>
-    """
-    return report
+# Gradio UI
+with gr.Blocks() as demo:
+    with gr.Row():
+        upload = gr.File(label="Upload CSV or Excel", type="file")
+        load_btn = gr.Button("Load & Analyze Data")
 
-def format_tuning_results(study, model, X, y):
-    """Format hyperparameter tuning results"""
-    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
-    y_pred = model.predict(X_test)
-    
-    report = f"""
-    <div style="font-family: Arial; max-width: 1000px; margin: 0 auto;">
-        <h1 style="color: #2B547E;">⚙️ Hyperparameter Tuning Results</h1>
-        
-        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-top: 20px;">
-            <div style="background: #f8f9fa; padding: 20px; border-radius: 8px;">
-                <h2 style="color: #2B547E;">📊 Best Parameters</h2>
-                <pre>{study.best_params}</pre>
-            </div>
-            
-            <div style="background: #f8f9fa; padding: 20px; border-radius: 8px;">
-                <h2 style="color: #2B547E;">📈 Performance Metrics</h2>
-                <p>Accuracy: {accuracy_score(y_test, y_pred):.4f}</p>
-                <p>Precision: {precision_score(y_test, y_pred, average='weighted'):.4f}</p>
-                <p>Recall: {recall_score(y_test, y_pred, average='weighted'):.4f}</p>
-                <p>F1 Score: {f1_score(y_test, y_pred, average='weighted'):.4f}</p>
-            </div>
-        </div>
-        
-        <div style="margin-top: 30px;">
-            <h2 style="color: #2B547E;">🔍 Classification Report</h2>
-            <pre>{classification_report(y_test, y_pred)}</pre>
-        </div>
-    </div>
-    """
-    return report
+    data_output = gr.DataFrame()
+    insights_output = gr.Textbox(label="Insights & Visuals (SmolAgent)", lines=15)
+    model_scores = gr.JSON(label="Model Performance Scores")
+    param_table = gr.DataFrame(label="Top 7 Hyperparameters")
+    shap_img = gr.Image(label="SHAP Plot")
+    lime_out = gr.HTML(label="LIME Explanation")
 
-# Create Gradio interface
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
-    gr.Markdown("# 🧠 Advanced Data Analysis with AI")
-    
-    with gr.Tab("Data Analysis"):
-        with gr.Row():
-            with gr.Column():
-                data_file = gr.File(label="Upload CSV", file_types=[".csv"])
-                notes = gr.Textbox(label="Analysis Notes (Optional)", lines=3)
-                analyze_btn = gr.Button("Analyze Data", variant="primary")
-            
-            with gr.Column():
-                analysis_report = gr.HTML(label="Analysis Report")
-                viz_gallery = gr.Gallery(label="Visualizations")
-    
-    with gr.Tab("Model Tuning"):
-        with gr.Row():
-            with gr.Column():
-                tune_file = gr.File(label="Upload CSV for Tuning", file_types=[".csv"])
-                trials = gr.Slider(10, 200, value=50, label="Number of Trials")
-                tune_btn = gr.Button("Tune Hyperparameters", variant="primary")
-            
-            with gr.Column():
-                tuning_report = gr.HTML(label="Tuning Results")
-                tuning_viz = gr.Gallery(label="Tuning Visualizations")
-    
-    # Event handlers
-    analyze_btn.click(
-        fn=analyze_data,
-        inputs=[data_file, notes],
-        outputs=[analysis_report, viz_gallery]
-    )
-    
-    tune_btn.click(
-        fn=tune_hyperparameters,
-        inputs=[tune_file, trials],
-        outputs=[tuning_report, tuning_viz]
-    )
+    load_btn.click(fn=load_data, inputs=upload, outputs=data_output)
+    load_btn.click(fn=get_insights, inputs=data_output, outputs=insights_output)
+    load_btn.click(fn=run_model, inputs=data_output, outputs=[model_scores, param_table])
+    load_btn.click(fn=explainability, inputs=data_output, outputs=[shap_img, lime_out])
 
-demo.launch()
+demo.launch()