Update app.py

app.py

@@ -12,16 +12,21 @@ import time
 import psutil
 import shutil
 import ast
+import seaborn as sns
+from sklearn.svm import SVC
 from smolagents import HfApiModel, CodeAgent
 from huggingface_hub import login
 from sklearn.model_selection import train_test_split, cross_val_score
-from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix, ConfusionMatrixDisplay
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, classification_report
+from sklearn.metrics import ConfusionMatrixDisplay
 from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
 from sklearn.linear_model import LogisticRegression
 from sklearn.preprocessing import LabelEncoder
 from datetime import datetime
 from PIL import Image
-from sklearn.svm import SVC
+from xgboost import XGBClassifier
+
+
 
 # Authenticate with Hugging Face
 hf_token = os.getenv("HF_TOKEN")
@@ -43,12 +48,12 @@ def clean_data(df):
 def upload_file(file):
     global df_global
     if file is None:
-        return pd.DataFrame({"Error": ["No file uploaded."]}), None
+        return pd.DataFrame({"Error": ["No file uploaded."]})
     ext = os.path.splitext(file.name)[-1]
     df = pd.read_csv(file.name) if ext == ".csv" else pd.read_excel(file.name)
     df = clean_data(df)
     df_global = df
-    return df.head(), df
+    return df.head()
 
 def format_analysis_report(raw_output, visuals):
     try:
@@ -154,117 +159,445 @@ def analyze_data(csv_file, additional_notes=""):
     run.finish()
     return format_analysis_report(analysis_result, visuals)
 
-def compare_models(selected_models, df):
-    if df is None or len(selected_models) == 0:
-        return pd.DataFrame(), []
-    target = df.columns[-1]
-    X = df.drop(target, axis=1)
-    y = df[target]
+def compare_models():
+    if df_global is None:
+        return "Please upload and preprocess a dataset first."
+    
+    target = df_global.columns[-1]
+    X = df_global.drop(target, axis=1)
+    y = df_global[target]
+
     if y.dtype == 'object':
         y = LabelEncoder().fit_transform(y)
-    model_dict = {
+
+    models = {
         "RandomForest": RandomForestClassifier(),
         "LogisticRegression": LogisticRegression(max_iter=1000),
-        "SVC": SVC(probability=True)
+        "SVC": SVC()
     }
+
     results = []
-    confusion_imgs = []
-    for name in selected_models:
-        model = model_dict[name]
-        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
+    for name, model in models.items():
+        scores = cross_val_score(model, X, y, cv=5)
+        results.append({
+            "Model": name,
+            "CV Mean Accuracy": np.mean(scores),
+            "CV Std Dev": np.std(scores)
+        })
+        wandb.log({f"{name}_cv_mean": np.mean(scores), f"{name}_cv_std": np.std(scores)})
+
+    results_df = pd.DataFrame(results)
+    return results_df
+    
+# 1. prepare_data should come first
+def prepare_data(df, target_column=None):
+    from sklearn.model_selection import train_test_split
+
+    # If no target column is specified, select the first object column or the last column
+    if target_column is None:
+        target_column = df.select_dtypes(include=['object']).columns[0] if len(df.select_dtypes(include=['object']).columns) > 0 else df.columns[-1]
+    
+    X = df.drop(columns=[target_column])
+    y = df[target_column]
+    
+    return train_test_split(X, y, test_size=0.3, random_state=42)
+
+
+def train_model(_):
+    try:
+        wandb.login(key=os.environ.get("WANDB_API_KEY"))
+        wandb_run = wandb.init(
+            project="huggingface-data-analysis",
+            name=f"Optuna_Run_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
+            reinit=True
+        )
+
+        X_train, X_test, y_train, y_test = prepare_data(df_global)
+
+        def objective(trial):
+            params = {
+                "n_estimators": trial.suggest_int("n_estimators", 50, 200),
+                "max_depth": trial.suggest_int("max_depth", 3, 10),
+            }
+            model = RandomForestClassifier()
+            score = cross_val_score(model, X_train, y_train, cv=3).mean()
+            wandb.log({**params, "cv_score": score})
+            return score
+
+        study = optuna.create_study(direction="maximize")
+        study.optimize(objective, n_trials=15)
+
+        best_params = study.best_params
+        model = RandomForestClassifier()
         model.fit(X_train, y_train)
         y_pred = model.predict(X_test)
-        y_proba = model.predict_proba(X_test)[:, 1] if hasattr(model, "predict_proba") and len(np.unique(y)) == 2 else None
+
+
         metrics = {
-            "Model": name,
-            "Accuracy": accuracy_score(y_test, y_pred),
-            "Precision": precision_score(y_test, y_pred, average="weighted", zero_division=0),
-            "Recall": recall_score(y_test, y_pred, average="weighted", zero_division=0),
-            "F1": f1_score(y_test, y_pred, average="weighted", zero_division=0),
-            "ROC-AUC": roc_auc_score(y_test, y_proba) if y_proba is not None else "N/A"
+            "accuracy": accuracy_score(y_test, y_pred),
+            "precision": precision_score(y_test, y_pred, average="weighted", zero_division=0),
+            "recall": recall_score(y_test, y_pred, average="weighted", zero_division=0),
+            "f1_score": f1_score(y_test, y_pred, average="weighted", zero_division=0),
         }
-        results.append(metrics)
-        # Confusion matrix plot
-        fig, ax = plt.subplots()
-        ConfusionMatrixDisplay.from_estimator(model, X_test, y_test, ax=ax)
-        img_path = f"conf_matrix_{name}.png"
-        plt.savefig(img_path)
-        confusion_imgs.append(img_path)
-        plt.close(fig)
-    results_df = pd.DataFrame(results)
-    return results_df, confusion_imgs
-
-def ab_test_models(model_a, model_b, df):
-    if df is None or model_a == model_b:
-        return pd.DataFrame()
-    target = df.columns[-1]
-    X = df.drop(target, axis=1)
-    y = df[target]
+        wandb.log(metrics)
+        wandb_run.finish()
+
+        # Top 7 trials
+        top_trials = sorted(study.trials, key=lambda x: x.value, reverse=True)[:7]
+        trial_rows = []
+        for t in top_trials:
+            row = t.params.copy()
+            row["score"] = t.value
+            trial_rows.append(row)
+        trials_df = pd.DataFrame(trial_rows)
+
+        return metrics, trials_df
+
+    except Exception as e:
+        print(f"Training Error: {e}")
+        return {}, pd.DataFrame()
+
+# Added a/b functions to existing code
+def create_model_comparison_plots(results_df):
+    """Create visualizations for model comparison results"""
+    os.makedirs('./comparison_plots', exist_ok=True)
+    plot_paths = []
+    
+    # Model performance comparison
+    plt.figure(figsize=(12, 6))
+    sns.barplot(data=results_df, x='Model', y='Test Accuracy')
+    plt.title('Model Accuracy Comparison')
+    plt.xticks(rotation=45)
+    accuracy_path = './comparison_plots/accuracy_comparison.png'
+    plt.savefig(accuracy_path, bbox_inches='tight')
+    plot_paths.append(accuracy_path)
+    plt.close()
+    
+    # Metric radar chart
+    metrics = ['Test Accuracy', 'Precision', 'Recall', 'F1 Score']
+    if not results_df['ROC AUC'].isna().all():
+        metrics.append('ROC AUC')
+    
+    plt.figure(figsize=(10, 10))
+    ax = plt.subplot(111, polar=True)
+    
+    angles = np.linspace(0, 2*np.pi, len(metrics), endpoint=False)
+    angles = np.concatenate((angles, [angles[0]]))
+    
+    for idx, row in results_df.iterrows():
+        values = row[metrics].values.flatten().tolist()
+        values += values[:1]
+        ax.plot(angles, values, 'o-', label=row['Model'])
+    
+    ax.set_thetagrids(angles[:-1] * 180/np.pi, metrics)
+    ax.set_title('Model Performance Radar Chart')
+    ax.legend(bbox_to_anchor=(1.1, 1.1))
+    radar_path = './comparison_plots/radar_chart.png'
+    plt.savefig(radar_path, bbox_inches='tight')
+    plot_paths.append(radar_path)
+    plt.close()
+    
+    return plot_paths
+
+def compare_models_enhanced():
+    """Enhanced model comparison with more metrics and visualizations"""
+    if df_global is None:
+        return "Please upload and preprocess a dataset first.", [], []
+    
+    target = df_global.columns[-1]
+    X = df_global.drop(target, axis=1)
+    y = df_global[target]
+
     if y.dtype == 'object':
         y = LabelEncoder().fit_transform(y)
+    
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
-    mid = len(X_test) // 2
-    X_a, X_b = X_test[:mid], X_test[mid:]
-    y_a, y_b = y_test[:mid], y_test[mid:]
-    model_dict = {
+    
+    # Define models to compare
+    models = {
         "RandomForest": RandomForestClassifier(),
         "LogisticRegression": LogisticRegression(max_iter=1000),
+        "GradientBoosting": GradientBoostingClassifier(),
+        "XGBoost": XGBClassifier(use_label_encoder=False, eval_metric='logloss'),
         "SVC": SVC(probability=True)
     }
+    
     results = []
-    for name, X_grp, y_grp in zip([model_a, model_b], [X_a, X_b], [y_a, y_b]):
-        model = model_dict[name]
+    
+    for name, model in models.items():
+        start_time = time.time()
+        
+        # Cross validation
+        cv_scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
+        
+        # Full training and test evaluation
         model.fit(X_train, y_train)
-        y_pred = model.predict(X_grp)
+        y_pred = model.predict(X_test)
+        y_proba = model.predict_proba(X_test)[:, 1] if hasattr(model, 'predict_proba') else None
+        
+        # Calculate metrics
         metrics = {
-            "Model": name,
-            "Accuracy": accuracy_score(y_grp, y_pred),
-            "Precision": precision_score(y_grp, y_pred, average="weighted", zero_division=0),
-            "Recall": recall_score(y_grp, y_pred, average="weighted", zero_division=0),
-            "F1": f1_score(y_grp, y_pred, average="weighted", zero_division=0),
+            'Model': name,
+            'CV Mean Accuracy': np.mean(cv_scores),
+            'CV Std Dev': np.std(cv_scores),
+            'Test Accuracy': accuracy_score(y_test, y_pred),
+            'Precision': precision_score(y_test, y_pred, average='weighted'),
+            'Recall': recall_score(y_test, y_pred, average='weighted'),
+            'F1 Score': f1_score(y_test, y_pred, average='weighted'),
+            'ROC AUC': roc_auc_score(y_test, y_proba) if y_proba is not None and len(np.unique(y_test)) == 2 else np.nan,
+            'Training Time (s)': time.time() - start_time
         }
+        
         results.append(metrics)
-    return pd.DataFrame(results)
-
-def get_model_choices():
-    return ["RandomForest", "LogisticRegression", "SVC"]
-
-def clear_confusion_imgs():
-    for name in get_model_choices():
-        img_path = f"conf_matrix_{name}.png"
-        if os.path.exists(img_path):
-            os.remove(img_path)
-
-def main():
-    with gr.Blocks() as demo:
-        gr.Markdown("# 🤖 Model Comparison & A/B Testing (Hugging Face + Gradio)")
-        with gr.Row():
-            with gr.Column():
-                file_input = gr.File(label="Upload CSV or Excel", type="filepath")
-                df_output = gr.DataFrame(label="Cleaned Data Preview")
-                state = gr.State()
-                file_input.change(fn=upload_file, inputs=file_input, outputs=[df_output, state])
-            with gr.Column():
-                model_choices = gr.CheckboxGroup(
-                    choices=get_model_choices(),
-                    value=["RandomForest", "LogisticRegression"],
-                    label="Select Models to Compare"
+        
+        # Log to wandb
+        if wandb.run:
+            wandb.log({f"{name}_{k}": v for k, v in metrics.items() if k != 'Model'})
+    
+    # Create visualizations
+    results_df = pd.DataFrame(results)
+    plot_paths = create_model_comparison_plots(results_df)
+    
+    return results_df, plot_paths
+
+def perform_ab_test(model_a_name, model_b_name):
+    """Perform A/B test between two specific models"""
+    if df_global is None:
+        return {"error": "Please upload and preprocess a dataset first."}, []
+    
+    target = df_global.columns[-1]
+    X = df_global.drop(target, axis=1)
+    y = df_global[target]
+
+    if y.dtype == 'object':
+        y = LabelEncoder().fit_transform(y)
+    
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
+    
+    # Define all available models
+    model_library = {
+        "RandomForest": RandomForestClassifier(),
+        "LogisticRegression": LogisticRegression(max_iter=1000),
+        "GradientBoosting": GradientBoostingClassifier(),
+        "XGBoost": XGBClassifier(use_label_encoder=False, eval_metric='logloss'),
+        "SVC": SVC(probability=True)
+    }
+    
+    # Get the selected models
+    model_a = model_library.get(model_a_name)
+    model_b = model_library.get(model_b_name)
+    
+    if not model_a or not model_b:
+        return {"error": "Invalid model selection"}, []
+    
+    # Train both models
+    model_a.fit(X_train, y_train)
+    model_b.fit(X_train, y_train)
+    
+    # Get predictions
+    y_pred_a = model_a.predict(X_test)
+    y_pred_b = model_b.predict(X_test)
+    
+    # Calculate metrics
+    metrics_a = {
+        'accuracy': accuracy_score(y_test, y_pred_a),
+        'precision': precision_score(y_test, y_pred_a, average='weighted'),
+        'recall': recall_score(y_test, y_pred_a, average='weighted'),
+        'f1': f1_score(y_test, y_pred_a, average='weighted')
+    }
+    
+    metrics_b = {
+        'accuracy': accuracy_score(y_test, y_pred_b),
+        'precision': precision_score(y_test, y_pred_b, average='weighted'),
+        'recall': recall_score(y_test, y_pred_b, average='weighted'),
+        'f1': f1_score(y_test, y_pred_b, average='weighted')
+    }
+    
+    # Calculate relative improvements
+    improvements = {
+        'accuracy_improvement': metrics_b['accuracy'] - metrics_a['accuracy'],
+        'f1_improvement': metrics_b['f1'] - metrics_a['f1'],
+        'relative_improvement': (metrics_b['accuracy'] - metrics_a['accuracy']) / metrics_a['accuracy'] if metrics_a['accuracy'] != 0 else 0
+    }
+    
+    # Create comparison DataFrame
+    comparison_df = pd.DataFrame({
+        'Metric': list(metrics_a.keys()),
+        model_a_name: list(metrics_a.values()),
+        model_b_name: list(metrics_b.values())
+    })
+    
+    # Log to wandb
+    if wandb.run:
+        wandb.log({
+            f"A_B_Test/{model_a_name}_metrics": metrics_a,
+            f"A_B_Test/{model_b_name}_metrics": metrics_b,
+            f"A_B_Test/Improvements": improvements
+        })
+    
+    # Create visualization
+    plt.figure(figsize=(10, 6))
+    comparison_df.set_index('Metric').plot(kind='bar', rot=0)
+    plt.title(f'A/B Test: {model_a_name} vs {model_b_name}')
+    plt.ylabel('Score')
+    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
+    plot_path = './comparison_plots/ab_test_comparison.png'
+    plt.savefig(plot_path, bbox_inches='tight')
+    plt.close()
+    
+    return {
+        'metrics_comparison': comparison_df.to_dict(),
+        'improvements': improvements
+    }, [plot_path]
+
+
+
+def explainability(_):
+    import warnings
+    warnings.filterwarnings("ignore")
+
+    target = df_global.columns[-1]
+    X = df_global.drop(target, axis=1)
+    y = df_global[target]
+
+    if y.dtype == "object":
+        y = LabelEncoder().fit_transform(y)
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+    model = RandomForestClassifier()
+    model.fit(X_train, y_train)
+
+    explainer = shap.TreeExplainer(model)
+    shap_values = explainer.shap_values(X_test)
+
+    try:
+        if isinstance(shap_values, list):
+            class_idx = 0
+            sv = shap_values[class_idx]
+        else:
+            sv = shap_values
+
+        # Ensure 2D input shape for SHAP plot
+        if len(sv.shape) > 2:
+            sv = sv.reshape(sv.shape[0], -1)  # Flatten any extra dimensions
+
+        # Use safe feature names if mismatch, fallback to dummy
+        num_features = sv.shape[1]
+        if num_features <= X_test.shape[1]:
+            feature_names = X_test.columns[:num_features]
+        else:
+            feature_names = [f"Feature_{i}" for i in range(num_features)]
+
+        X_shap_safe = pd.DataFrame(np.zeros_like(sv), columns=feature_names)
+
+        shap.summary_plot(sv, X_shap_safe, show=False)
+        shap_path = "./shap_plot.png"
+        plt.title("SHAP Summary")
+        plt.savefig(shap_path)
+        if wandb.run:
+            wandb.log({"shap_summary": wandb.Image(shap_path)})
+        plt.clf()
+
+    except Exception as e:
+        shap_path = "./shap_error.png"
+        print("SHAP plotting failed:", e)
+        plt.figure(figsize=(6, 3))
+        plt.text(0.5, 0.5, f"SHAP Error:\n{str(e)}", ha='center', va='center')
+        plt.axis('off')
+        plt.savefig(shap_path)
+        if wandb.run:
+            wandb.log({"shap_error": wandb.Image(shap_path)})
+        plt.clf()
+
+    # LIME
+    lime_explainer = lime.lime_tabular.LimeTabularExplainer(
+        X_train.values,
+        feature_names=X_train.columns.tolist(),
+        class_names=[str(c) for c in np.unique(y_train)],
+        mode='classification'
+    )
+    lime_exp = lime_explainer.explain_instance(X_test.iloc[0].values, model.predict_proba)
+    lime_fig = lime_exp.as_pyplot_figure()
+    lime_path = "./lime_plot.png"
+    lime_fig.savefig(lime_path)
+    if wandb.run:
+        wandb.log({"lime_explanation": wandb.Image(lime_path)})
+    plt.clf()
+
+    return shap_path, lime_path
+
+with gr.Blocks() as demo:
+    gr.Markdown("## 📊 AI-Powered Data Analysis with Hyperparameter Optimization")
+
+    with gr.Row():
+        with gr.Column():
+            file_input = gr.File(label="Upload CSV or Excel", type="filepath")
+            df_output = gr.DataFrame(label="Cleaned Data Preview")
+            file_input.change(fn=upload_file, inputs=file_input, outputs=df_output)
+
+        with gr.Column():
+            insights_output = gr.HTML(label="Insights from SmolAgent")
+            visual_output = gr.Gallery(label="Visualizations (Auto-generated by Agent)", columns=2)
+            agent_btn = gr.Button("Run AI Agent (5 Insights + 5 Visualizations)")
+
+    with gr.Row():
+        train_btn = gr.Button("Train Model with Optuna + WandB")
+        metrics_output = gr.JSON(label="Performance Metrics")
+        trials_output = gr.DataFrame(label="Top 7 Hyperparameter Trials")
+
+    with gr.Row():
+        explain_btn = gr.Button("SHAP + LIME Explainability")
+        shap_img = gr.Image(label="SHAP Summary Plot")
+        lime_img = gr.Image(label="LIME Explanation")
+  
+    # Add new A/B testing components
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("### 🆚 Model A/B Testing")
+            with gr.Row():
+                model_a_select = gr.Dropdown(
+                    choices=["RandomForest", "LogisticRegression", "GradientBoosting", "XGBoost", "SVC"],
+                    label="Select Model A",
+                    value="RandomForest"
+                )
+                model_b_select = gr.Dropdown(
+                    choices=["RandomForest", "LogisticRegression", "GradientBoosting", "XGBoost", "SVC"],
+                    label="Select Model B",
+                    value="LogisticRegression"
                 )
-                compare_btn = gr.Button("Compare Models")
-                metrics_output = gr.DataFrame(label="Model Performance Metrics")
-                confusion_gallery = gr.Gallery(label="Confusion Matrices", columns=3)
-                compare_btn.click(fn=compare_models, inputs=[model_choices, state], outputs=[metrics_output, confusion_gallery])
-        gr.Markdown("## A/B Test: Compare Two Models on Test Set")
-        with gr.Row():
-            ab_model_a = gr.Dropdown(get_model_choices(), value="RandomForest", label="Model A")
-            ab_model_b = gr.Dropdown(get_model_choices(), value="LogisticRegression", label="Model B")
-            ab_btn = gr.Button("Run A/B Test")
-            ab_output = gr.DataFrame(label="A/B Test Results")
-            ab_btn.click(fn=ab_test_models, inputs=[ab_model_a, ab_model_b, state], outputs=ab_output)
-        gr.Markdown("---\nBuilt for Hugging Face Spaces with Gradio. Upload your data, select models, and compare!")
-    return demo
-
-if __name__ == "__main__":
-    clear_confusion_imgs()
-    demo = main()
-    demo.launch()
+            ab_test_btn = gr.Button("Run A/B Test")
+        
+        with gr.Column():
+            ab_test_results = gr.JSON(label="A/B Test Results")
+            ab_test_plots = gr.Gallery(label="A/B Test Visualizations")
+
+    # Add model comparison components
+    with gr.Row():
+        compare_btn = gr.Button("Compare All Models")
+        comparison_results = gr.DataFrame(label="Model Comparison Results")
+        comparison_plots = gr.Gallery(label="Comparison Visualizations")
+
+
+
+    
+    agent_btn.click(fn=analyze_data, inputs=[file_input], outputs=[insights_output, visual_output])
+    train_btn.click(fn=train_model, inputs=[file_input], outputs=[metrics_output, trials_output])
+    explain_btn.click(fn=explainability, inputs=[], outputs=[shap_img, lime_img])
+
+# New handlers for A/B testing and comparison
+    ab_test_btn.click(
+        fn=perform_ab_test,
+        inputs=[model_a_select, model_b_select],
+        outputs=[ab_test_results, ab_test_plots]
+    )
+    
+    compare_btn.click(
+        fn=compare_models_enhanced,
+        inputs=[],
+        outputs=[comparison_results, comparison_plots]
+    )
+
+
+
+demo.launch(debug=True)