Update app.py

app.py

@@ -1,213 +1,61 @@
 import streamlit as st
-import pandas as pd
+import tensorflow as tf
+from tensorflow import keras
 import numpy as np
+import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
-from ydata_profiling import ProfileReport
-from streamlit_pandas_profiling import st_profile_report
-import os
-from datetime import datetime
-import re
-import tempfile
-from scipy import stats
-from sklearn.impute import SimpleImputer
-from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder
-from sklearn.decomposition import PCA
-import streamlit.components.v1 as components
-from io import StringIO
-import tensorflow as tf
-from tensorflow import keras
 from sklearn.model_selection import train_test_split, GridSearchCV
+from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder
+from sklearn.decomposition import PCA, TruncatedSVD
+from sklearn.manifold import TSNE
+import umap.umap_ as umap
+import shap
+import joblib
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_curve, auc, mean_squared_error, mean_absolute_error, r2_score, classification_report
+from sklearn.pipeline import Pipeline
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
 from sklearn.cluster import KMeans, DBSCAN
 from sklearn.mixture import GaussianMixture
 from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
 from sklearn.linear_model import LogisticRegression, LinearRegression
 from xgboost import XGBClassifier, XGBRegressor
-from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_curve, auc, mean_squared_error, mean_absolute_error, r2_score
+import matplotlib.pyplot as plt
+from io import BytesIO
 import time
 
-# Custom CSS for modern styling (as before)
-st.markdown("""
-    <style>
-    @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600&display=swap');
-
-    /* Base styles */
-    html {
-        font-family: 'Inter', sans-serif;
-        scroll-behavior: smooth;
-    }
-
-    /* Main container */
-    .stApp {
-        background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%);
-        color: #2d3436;
-    }
-
-    /* Sidebar styling */
-    .st-emotion-cache-6qob1r {
-        background: linear-gradient(195deg, #2d3436 0%, #1a1e1f 100%) !important;
-        border-right: 1px solid rgba(255,255,255,0.1) !important;
-    }
-
-    /* Data-Vision Pro title in sidebar (white) */
-    .st-emotion-cache-6qob1r .stMarkdown h1 {
-        color: white !important;
-    }
-
-    /* Navigation dropdown text black */
-    .st-emotion-cache-6qob1r .stSelectbox label,
-    .st-emotion-cache-6qob1r .stSelectbox div[data-baseweb="select"] > div {
-        color: black !important;
-    }
-
-    /* Note text red */
-    .st-emotion-cache-6qob1r .stMarkdown p:has(> em) {
-        color: #ff4b4b !important;
-        font-size: 0.9rem !important;
-        margin: 0.25rem 0 !important;
-    }
-
-    /* Green links in sidebar */
-    .st-emotion-cache-6qob1r .stMarkdown a {
-        color: #4CAF50 !important;
-    }
-
-    /* Footer text white */
-    .st-emotion-cache-6qob1r .stMarkdown p {
-        color: white !important;
-        font-size: 0.9rem !important;
-        margin: 0.25rem 0 !important;
-    }
-
-    /* Footer name bold */
-    .st-emotion-cache-6qob1r .stMarkdown p strong {
-        font-weight: 600 !important;
-    }
-
-    /* Footer spacing and border */
-    .st-emotion-cache-6qob1r .stMarkdown:has(> p > strong) {
-        margin-top: 2rem !important;
-        padding-top: 1rem !important;
-        border-top: 1px solid rgba(255,255,255,0.1) !important;
-    }
-
-    /* Improved selectboxes */
-    div[data-baseweb="select"] {
-        border: 1px solid #ced4da;
-        border-radius: 0.25rem;
-        padding: 0.375rem 0.75rem;
-        transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;
-    }
-
-    div[data-baseweb="select"]:focus,
-    div[data-baseweb="select"]:hover {
-        border-color: #80bdff;
-        box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25);
-    }
-
-    /* Metric cards */
-    div[data-testid="metric-container"] {
-        background-color: #fff;
-        border-radius: 0.5rem;
-        padding: 1rem;
-        box-shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075);
-        transition: all 0.3s ease;
-    }
-
-    div[data-testid="metric-container"]:hover {
-        transform: translateY(-0.25rem);
-        box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.1);
-    }
-
-    /* Main content layout */
-    .stApp {
-        max-width: 1200px;
-        margin: 0 auto;
-    }
-    </style>
-""", unsafe_allow_html=True)
-
-# Animated elements JavaScript (as before)
-components.html("""
-    <script>
-    document.querySelectorAll('[data-testid="metric-container"]').forEach(card => {
-        card.style.transition = 'transform 0.3s ease, box-shadow 0.3s ease';
-        card.addEventListener('mouseover', () => {
-            card.style.transform = 'translateY(-4px)';
-            card.style.boxShadow = '0 8px 16px rgba(0,0,0,0.1)';
-        });
-        card.addEventListener('mouseout', () => {
-            card.style.transform = 'none';
-            card.style.boxShadow = '0 4px 6px rgba(0,0,0,0.05)';
-        });
-    });
-    </script>
-""", height=0)
-
-# Helper Functions (as before, but updated train_model and evaluate_model and added StreamlitCallback)
-def enhance_section_title(title):
-    st.markdown(f"<h2 style='border-bottom: 2px solid #ccc; padding-bottom: 5px;'>{title}</h2>", unsafe_allow_html=True)
-
-def update_cleaned_data(df):
-    st.session_state.cleaned_data = df
-    if 'data_versions' not in st.session_state:
-        st.session_state.data_versions = [st.session_state.raw_data.copy()]
-    st.session_state.data_versions.append(df.copy())
-    st.success("✅ Action completed successfully!")
-    st.rerun()
-
-def generate_quality_report(df):
-    report = {
-        'basic': {
-            'rows': df.shape[0],
-            'columns': df.shape[1],
-            'missing': df.isna().sum().sum(),
-            'duplicates': df.duplicated().sum()
-        },
-        'columns': {}
-    }
-    for col in df.columns:
-        col_report = {
-            'type': str(df[col].dtype),
-            'unique': df[col].nunique(),
-            'missing': df[col].isna().sum(),
-        }
-        if pd.api.types.is_numeric_dtype(df[col]):
-            col_report.update({
-                'mean': df[col].mean(),
-                'std': df[col].std(),
-                'zeros': (df[col] == 0).sum()
-            })
-        report['columns'][col] = col_report
-    return report
+# Set page config
+st.set_page_config(page_title="Neural-Vision Enhanced", layout="wide")
 
+# Helper Functions
 def get_model_config(model_type, problem_type):
     configs = {
         "Random Forest": {
-            "Regression": {"model_class": RandomForestRegressor, "params": {"n_estimators": 100},
+            "Regression": {"model_class": RandomForestRegressor, "params": {"n_estimators": 100, "random_state": 42},
                            "grid_params": {"n_estimators": [50, 100, 200], "max_depth": [None, 10, 20]}},
-            "Binary Classification": {"model_class": RandomForestClassifier, "params": {"n_estimators": 100},
+            "Binary Classification": {"model_class": RandomForestClassifier, "params": {"n_estimators": 100, "random_state": 42},
                                      "grid_params": {"n_estimators": [50, 100, 200], "max_depth": [None, 10, 20]}},
-            "Multi-Class": {"model_class": RandomForestClassifier, "params": {"n_estimators": 100},
+            "Multi-Class": {"model_class": RandomForestClassifier, "params": {"n_estimators": 100, "random_state": 42},
                             "grid_params": {"n_estimators": [50, 100, 200], "max_depth": [None, 10, 20]}}
         },
         "XGBoost": {
-            "Regression": {"model_class": XGBRegressor, "params": {"n_estimators": 100},
+            "Regression": {"model_class": XGBRegressor, "params": {"n_estimators": 100, "random_state": 42},
                            "grid_params": {"n_estimators": [50, 100, 200], "max_depth": [3, 5, 7], "learning_rate": [0.01, 0.1, 0.3]}},
-            "Binary Classification": {"model_class": XGBClassifier, "params": {"n_estimators": 100},
+            "Binary Classification": {"model_class": XGBClassifier, "params": {"n_estimators": 100, "random_state": 42, "use_label_encoder": False, "eval_metric": 'logloss'},
                                      "grid_params": {"n_estimators": [50, 100, 200], "max_depth": [3, 5, 7], "learning_rate": [0.01, 0.1, 0.3]}},
-            "Multi-Class": {"model_class": XGBClassifier, "params": {"n_estimators": 100},
+            "Multi-Class": {"model_class": XGBClassifier, "params": {"n_estimators": 100, "random_state": 42, "use_label_encoder": False, "eval_metric": 'mlogloss'},
                             "grid_params": {"n_estimators": [50, 100, 200], "max_depth": [3, 5, 7], "learning_rate": [0.01, 0.1, 0.3]}}
         },
         "Logistic Regression": {
-            "Binary Classification": {"model_class": LogisticRegression, "params": {"max_iter": 1000},
+            "Binary Classification": {"model_class": LogisticRegression, "params": {"max_iter": 1000, "random_state": 42},
                                      "grid_params": {"C": [0.1, 1.0, 10.0], "solver": ["lbfgs", "liblinear"]}}
         },
         "Linear Regression": {
             "Regression": {"model_class": LinearRegression, "params": {}, "grid_params": {}}
         },
         "K-Means": {
-            "Clustering": {"model_class": KMeans, "params": {"n_clusters": 3},
+            "Clustering": {"model_class": KMeans, "params": {"n_clusters": 3, "random_state": 42},
                            "grid_params": {"n_clusters": [2, 3, 4, 5]}}
         },
         "DBSCAN": {
@@ -215,13 +63,14 @@ def get_model_config(model_type, problem_type):
                            "grid_params": {"eps": [0.3, 0.5, 0.7], "min_samples": [3, 5, 10]}}
         },
         "Gaussian Mixture": {
-            "Clustering": {"model_class": GaussianMixture, "params": {"n_components": 3},
+            "Clustering": {"model_class": GaussianMixture, "params": {"n_components": 3, "random_state": 42},
                            "grid_params": {"n_components": [2, 3, 4, 5]}}
         }
     }
     return configs.get(model_type, {}).get(problem_type, {"model_class": None, "params": {}, "grid_params": {}})
 
 def preprocess_data(X_train, X_test, numerical_features, categorical_features):
+    # Define the numeric and categorical transformers
     numeric_transformer = Pipeline(steps=[
         ('imputer', SimpleImputer(strategy='mean')),
         ('scaler', StandardScaler())])
@@ -230,16 +79,22 @@ def preprocess_data(X_train, X_test, numerical_features, categorical_features):
         ('imputer', SimpleImputer(strategy='most_frequent')),
         ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))])
 
+    # Combine transformers into a ColumnTransformer
     preprocessor = ColumnTransformer(
         transformers=[
             ('num', numeric_transformer, numerical_features),
             ('cat', categorical_transformer, categorical_features)],
-        remainder='drop')
+        remainder='passthrough') # Handle unseen columns
 
+    # Fit and transform the training data
     X_train_processed = preprocessor.fit_transform(X_train)
+
+    # Transform the test data
     X_test_processed = preprocessor.transform(X_test)
 
+    # Get feature names after one-hot encoding
     if categorical_features:
+        # Access the fitted OneHotEncoder
         onehot_encoder = preprocessor.named_transformers_['cat'].named_steps['onehot']
         categorical_feature_names = onehot_encoder.get_feature_names_out(categorical_features)
         feature_names = numerical_features + list(categorical_feature_names)
@@ -266,49 +121,41 @@ def build_neural_network(input_shape, output_units, problem_type, layers_config,
                   metrics=["mae" if problem_type == "Regression" else "accuracy"])
     return model
 
-class StreamlitCallback(keras.callbacks.Callback):
-    def __init__(self, chart_placeholder, metrics_placeholder):
-        super().__init__()
-        self.chart_placeholder = chart_placeholder
-        self.metrics_placeholder = metrics_placeholder
-        self.logs_data = []
-
-    def on_epoch_end(self, epoch, logs=None):
-        self.logs_data.append(logs)
-        hist_df = pd.DataFrame(self.logs_data)
-        with self.chart_placeholder.container(): # Use container for smoother updates
-            fig_hist = px.line(hist_df, x=hist_df.index, y=hist_df.columns, labels={'index': 'Epoch', 'value': 'Metric'})
-            st.plotly_chart(fig_hist, use_container_width=True)
-
-        with self.metrics_placeholder.container():
-            col_metrics = st.columns(len(logs))
-            for idx, (metric_name, metric_value) in enumerate(logs.items()):
-                col_metrics[idx].metric(metric_name.capitalize(), f"{metric_value:.4f}" if isinstance(metric_value, float) else metric_value)
-
-
-def train_model(model, X_train, y_train, X_test, y_test, epochs, batch_size, problem_type, callback, do_grid_search=False, params=None, grid_params=None): # Added callback
+def train_model(model, X_train, y_train, X_test, y_test, epochs, batch_size, problem_type, do_grid_search=False, params=None, grid_params=None, training_placeholder=None):
     start_time = time.time()
     history = None
-
     if isinstance(model, keras.Model):
-        history = model.fit(
-            X_train,
-            y_train,
-            epochs=epochs,
-            batch_size=batch_size,
-            validation_data=(X_test, y_test),
-            verbose=0,
-            callbacks=[callback] # Pass the callback here
-        )
+        # Define a callback to update Streamlit during training
+        class StreamlitCallback(keras.callbacks.Callback):
+            def __init__(self, placeholder):
+                super().__init__()
+                self.placeholder = placeholder
+                self.epoch_data = []
+
+            def on_epoch_end(self, epoch, logs=None):
+                self.epoch_data.append(logs)
+                df = pd.DataFrame(self.epoch_data)
+                fig = px.line(df, x=df.index, y=['loss', 'val_loss'], labels={'index': 'Epoch', 'value': 'Loss'})
+                fig.add_trace(go.Scatter(x=df.index, y=df['accuracy'], mode='lines', name='accuracy'))
+                fig.add_trace(go.Scatter(x=df.index, y=df['val_accuracy'], mode='lines', name='val_accuracy'))
+
+                self.placeholder.plotly_chart(fig)
+
+        streamlit_callback = StreamlitCallback(training_placeholder)
+        history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,
+                              validation_data=(X_test, y_test), verbose=0,
+                              callbacks=[streamlit_callback])
 
     else:
         if do_grid_search and grid_params:
-            grid_search = GridSearchCV(model, grid_params, cv=3, n_jobs=-1)
+            grid_search = GridSearchCV(model, grid_params, cv=3, n_jobs=-1, scoring='accuracy' if problem_type in ["Binary Classification", "Multi-Class"] else 'neg_mean_squared_error') # Added scoring
             grid_search.fit(X_train, y_train)
             model = grid_search.best_estimator_
+            st.write("Best parameters found by Grid Search:", grid_search.best_params_) # Print best params
         else:
             model.set_params(**params)
             model.fit(X_train, y_train)
+            history = None # Reset to none here
 
     training_time = time.time() - start_time
     return history, model, training_time
@@ -321,6 +168,7 @@ def evaluate_model(model, X_test, y_test, problem_type, le=None):
         metrics['mae'] = mean_absolute_error(y_test, y_pred)
         metrics['rmse'] = np.sqrt(metrics['mse'])
         metrics['r2'] = r2_score(y_test, y_pred)
+        return metrics, y_pred.flatten()
     elif problem_type in ["Binary Classification", "Multi-Class"]:
         y_pred_classes = (y_pred > 0.5).astype(int).flatten() if problem_type == "Binary Classification" else np.argmax(y_pred, axis=1)
         y_test_classes = y_test if problem_type == "Binary Classification" else np.argmax(y_test, axis=1)
@@ -328,12 +176,10 @@ def evaluate_model(model, X_test, y_test, problem_type, le=None):
         metrics['precision'] = precision_score(y_test_classes, y_pred_classes, average='weighted', zero_division=0)
         metrics['recall'] = recall_score(y_test_classes, y_pred_classes, average='weighted', zero_division=0)
         metrics['f1'] = f1_score(y_test_classes, y_pred_classes, average='weighted', zero_division=0)
+        return metrics, y_pred_classes
     elif problem_type == "Clustering":
         labels = model.labels_ if hasattr(model, 'labels_') else model.predict(X_test)
-        metrics["n_clusters"] = len(np.unique(labels))
-
-    return metrics, y_pred if problem_type != "Clustering" else labels
-
+        return {"n_clusters": len(np.unique(labels))}, labels
 
 def save_model(model, preprocessor, features, target, problem_type, filename="model.pkl"):
     model_data = {
@@ -356,148 +202,43 @@ def load_model(model_file):
         model_data['model'] = keras.models.load_model(model_data['model_path'])
     return model_data
 
-
-# Sidebar Navigation (as before)
+# Sidebar Navigation
 with st.sidebar:
-    st.title("🔮 Data-Vision Pro")
-    st.markdown("Your AI-powered data analysis suite.")
+    st.title("🔮 Neural-Vision Enhanced")
+    st.markdown("Your AI-powered model toolbox.")
     st.markdown("---")
-    app_mode = st.selectbox(
-        "Navigation",
-        ["Data Upload", "Data Cleaning", "EDA", "Model Training"], # Added Model Training to Navigation
-        format_func=lambda x: f"📌 {x}"
-    )
-    if app_mode == "Data Upload":
-        st.info("⬆️ Upload your CSV or XLSX dataset to begin.")
-    elif app_mode == "Data Cleaning":
-        st.info("🧹 Clean and preprocess your data using various tools.")
-    elif app_mode == "EDA":
-        st.info("🔍 Explore your data visually and statistically.")
-    elif app_mode == "Model Training": # Info for Model Training
-        st.info("🧠 Train and evaluate machine learning models.")
-
+    app_mode = st.selectbox("Navigation", ["Data Upload", "Model Training", "Validation & Exploration"])
     st.markdown("---")
-    st.markdown("**Note**: This app requires `ydata-profiling`, `tensorflow`, `scikit-learn`, etc.")
-
-    # Download button (as before)
-    if 'cleaned_data' in st.session_state:
-        csv = st.session_state.cleaned_data.to_csv(index=False)
-        st.download_button(
-            label="Download Cleaned Data as CSV",
-            data=csv,
-            file_name='cleaned_data.csv',
-            mime='text/csv',
-        )
-
-    st.markdown("Created by Calvin Allen-Crawford")
-    st.markdown("v1.0 | © 2025")
-
-# Main App Pages (Data Upload, Data Cleaning, EDA - unchanged)
+    st.markdown("**Dependencies**: `tensorflow`, `shap`, `umap-learn`, `joblib`, `scikit-learn`, `plotly`, `xgboost`")
+    st.markdown("Created by Calvin Allen-Crawford | v1.2 | © 2025")
+
+# Main App Sections
 if app_mode == "Data Upload":
-    st.title("📤 Data Upload & Profiling")
-    st.header("Upload Your Dataset")
-    st.write("Supported formats: CSV, XLSX")
-    uploaded_file = st.file_uploader("Choose a file", type=["csv", "xlsx"], key="file_uploader")
-    if uploaded_file:
-        st.session_state.pop('raw_data', None)
-        st.session_state.pop('cleaned_data', None)
-        st.session_state.pop('data_versions', None)
-        st.session_state.pop('trained_model', None)
-        st.session_state.pop('feature_names_model', None)
-        st.session_state.pop('layers', None)
-        st.session_state.pop('presets', None)
-
-        try:
-            if uploaded_file.name.endswith('.csv'):
-                df = pd.read_csv(uploaded_file)
-            else:
-                df = pd.read_excel(uploaded_file)
-            if df.empty:
-                st.error("Uploaded file is empty. Please upload a valid dataset.")
-                st.stop()
-            st.session_state.raw_data = df
-            if 'data_versions' not in st.session_state:
-                st.session_state.data_versions = [df.copy()]
-            col1, col2, col3 = st.columns(3)
-            with col1: st.metric("Rows", df.shape[0])
-            with col2: st.metric("Columns", df.shape[1])
-            with col3: st.metric("Missing Values", df.isna().sum().sum())
-            if st.checkbox("Show Data Preview"):
-                st.dataframe(df.head(10), use_container_width=True)
-            if st.button("Generate Full Profile Report"):
-                with st.spinner("Generating report..."):
-                    pr = ProfileReport(df, explorative=True)
-                    st_profile_report(pr)
-            st.success("✅ Data loaded and profiled successfully!")
-        except Exception as e:
-            st.error(f"An error occurred: {str(e)}")
-
-elif app_mode == "Data Cleaning":
-    st.title("🧹 Smart Data Cleaning")
-    st.header("Preprocess and Transform Your Data")
-    if 'raw_data' not in st.session_state:
-        st.warning("Please upload data first in the Data Upload section.")
-        st.stop()
-    if 'cleaned_data' not in st.session_state:
-        st.session_state.cleaned_data = st.session_state.raw_data.copy()
-    df = st.session_state.cleaned_data.copy()
+    st.title("📤 Data Upload")
+    col1, col2, col3 = st.columns([1, 2, 1])
+    with col2:
+        uploaded_file = st.file_uploader("Upload CSV Dataset", type=["csv"])
 
-    enhance_section_title("📊 Data Health Dashboard")
-    with st.expander("Explore Data Health Metrics", expanded=True):
+    if uploaded_file:
+        df = pd.read_csv(uploaded_file)
+        st.session_state.df = df
+        st.write("---")
+        st.subheader("Dataset Preview")
+        st.dataframe(df.head(10))
+        st.write("---")
+        st.subheader("Statistics")
         col1, col2, col3 = st.columns(3)
-        with col1: st.metric("Columns", len(df.columns))
-        with col2: st.metric("Rows", len(df))
+        with col1: st.metric("Rows", df.shape[0])
+        with col2: st.metric("Columns", df.shape[1])
         with col3: st.metric("Missing Values", df.isna().sum().sum())
-        if st.button("Generate Detailed Health Report"):
-            with st.spinner("Generating report..."):
-                profile = ProfileReport(df, minimal=True)
-                st_profile_report(profile)
-        if 'data_versions' in st.session_state and len(st.session_state.data_versions) > 1:
-            if st.button("Undo Last Action"):
-                st.session_state.data_versions.pop()
-                st.session_state.cleaned_data = st.session_state.data_versions[-1].copy()
-                st.rerun()
-
-    with st.expander("🛠️ Data Cleaning Operations", expanded=True):
-        # ... (Data Cleaning operations - unchanged) ...
-        enhance_section_title("📊 Principal Component Analysis (PCA)")
-        numerical_cols = df.select_dtypes(include=np.number).columns.tolist()
-        if numerical_cols:
-            pca_cols = st.multiselect("Select columns for PCA", numerical_cols, default=numerical_cols)
-            if pca_cols:
-                st.subheader("Covariance Matrix Heatmap")
-                cov_matrix = df[pca_cols].cov()
-                fig_cov = px.imshow(cov_matrix, labels=dict(x="Features", y="Features", color="Covariance"), color_continuous_scale='RdBu_r')
-                st.plotly_chart(fig_cov)
-                n_components = st.slider("Number of components", 1, min(len(pca_cols), 10), 2)
-                if st.button("Apply PCA"):
-                    new_df = df.copy()
-                    scaler = StandardScaler()
-                    scaled_data = scaler.fit_transform(new_df[pca_cols])
-                    pca = PCA(n_components=n_components)
-                    pca_result = pca.fit_transform(scaled_data)
-                    pca_df = pd.DataFrame(pca_result, columns=[f'PC{i+1}' for i in range(n_components)])
-                    update_cleaned_data(pca_df.reset_index(drop=True))
-                    st.write("Explained Variance Ratio:", pca.explained_variance_ratio_)
-        else:
-            st.warning("No numerical columns available for PCA.")
-
-elif app_mode == "EDA":
-    st.title("🔍 Interactive Data Explorer")
-    # ... (EDA section - unchanged) ...
-    if fig:
-        fig.update_layout(template="plotly_white")
-        st.plotly_chart(fig, use_container_width=True)
-    else:
-        st.error("Please provide required inputs for the selected plot type.")
 
 elif app_mode == "Model Training":
     st.title("🧠 Model Training")
-    if 'cleaned_data' not in st.session_state:
-        st.warning("Please upload and clean data first in the Data Upload and Data Cleaning sections.")
+    if 'df' not in st.session_state:
+        st.warning("Please upload a dataset first.")
         st.stop()
 
-    df = st.session_state.cleaned_data.copy()
+    df = st.session_state.df
     problem_type = st.selectbox("Problem Type", ["Regression", "Binary Classification", "Multi-Class", "Clustering"])
     features = st.multiselect("Select Features", df.columns)
     target = st.selectbox("Select Target", df.columns) if problem_type != "Clustering" else None
@@ -554,8 +295,9 @@ elif app_mode == "Model Training":
                     param_name,
                     min_value=float(min(param_values)),
                     max_value=float(max(param_values)),
-                    value=float(param_values[1]))
+                    value=float(param_values[1])
 
+                # CAST TO INT FOR INTEGER PARAMETERS
                 if param_name in {'n_estimators', 'n_clusters', 'min_samples',
                                 'n_components', 'max_depth'}:
                     params[param_name] = int(slider_value)
@@ -574,7 +316,7 @@ elif app_mode == "Model Training":
     base_model = keras.models.load_model(uploaded_model) if uploaded_model else None
 
     if st.button("Train Model"):
-        with st.spinner("Starting Training..."): # Initial spinner message
+        with st.spinner("Preparing data..."):
             X = df[features]
             y = df[target] if problem_type != "Clustering" else None
             X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) if problem_type != "Clustering" else (X, X.copy(), None, None)
@@ -591,41 +333,27 @@ elif app_mode == "Model Training":
                     y_train = tf.keras.utils.to_categorical(y_train)
                     y_test = tf.keras.utils.to_categorical(y_test)
 
+        with st.spinner("Training model..."):
+            training_placeholder = st.empty()  # Placeholder for real-time training updates
             if model_type == "Neural Network":
                 if not layers_config and not base_model:
                     st.error("Please add layers or upload a pre-trained model.")
                     st.stop()
-                if problem_type == "Multi-Class":
-                    output_units = y_train.shape[1]
-                else:
-                    output_units = 1
-                model = build_neural_network(X_train_processed.shape[1:], output_units,
-                                            problem_type, layers_config, "Adam", learning_rate)
-
-                # Create placeholders for chart and metrics
-                training_chart_placeholder = st.empty() # Use st.empty()
-                training_metrics_placeholder = st.empty()
-
-                # Create StreamlitCallback instance
-                callback = StreamlitCallback(training_chart_placeholder, training_metrics_placeholder)
-
-                history, model, training_time = train_model(
-                    model,
-                    X_train_processed,
-                    y_train,
-                    X_test_processed,
-                    y_test,
-                    epochs,
-                    batch_size,
-                    problem_type,
-                    callback # Pass the callback to train_model
-                )
-
-            else: # Non-NN Models - No real-time visualization
+                model = base_model if base_model else build_neural_network(X_train_processed.shape[1:], y_train.shape[1] if problem_type == "Multi-Class" else 1,
+                                                problem_type, layers_config, "Adam", learning_rate)
+                history, model, training_time = train_model(model, X_train_processed, y_train, X_test_processed, y_test, epochs, batch_size, problem_type,
+                                                             training_placeholder=training_placeholder)
+            else:
                 config = get_model_config(model_type, problem_type)
                 model = config['model_class'](**config['params'])
                 history, model, training_time = train_model(model, X_train_processed, y_train, X_test_processed, y_test, epochs, batch_size, problem_type,
-                                                            None, params, config['grid_params']) # No callback for other models
+                                                            do_grid_search, params, config['grid_params'])
+                st.subheader("Training Metrics")
+                if history:
+                  fig = px.line(x=range(len(history.history['loss'])), y=history.history['loss'], labels={'x':'Epoch', 'y':'Loss'})
+                  st.plotly_chart(fig)
+                else:
+                  st.write("No history available for this model type.")
 
             st.session_state.model = model
             st.session_state.preprocessor = preprocessor
@@ -639,9 +367,122 @@ elif app_mode == "Model Training":
                 st.download_button("Download Model", f, file_name=filename)
             st.success(f"Model trained in {training_time:.2f}s and saved!")
 
-            # Model Evaluation and Metrics Display (for all model types)
-            st.subheader("Model Evaluation Metrics")
-            metrics, _ = evaluate_model(model, X_test_processed, y_test, problem_type, le)
-            metric_cols = st.columns(len(metrics))
-            for idx, (metric_name, metric_value) in enumerate(metrics.items()):
-                metric_cols[idx].metric(metric_name.capitalize(), f"{metric_value:.4f}" if isinstance(metric_value, float) else metric_value)
+elif app_mode == "Validation & Exploration":
+    st.title("🔍 Validation & Exploration")
+    if 'model' not in st.session_state or 'df' not in st.session_state:
+        st.warning("Please upload a dataset and train a model first.")
+        st.stop()
+
+    df = st.session_state.df
+    model = st.session_state.model
+    preprocessor = st.session_state.preprocessor
+    features = st.session_state.features
+    target = st.session_state.target
+    problem_type = st.session_state.problem_type
+    le = st.session_state.le
+
+    X = df[features]
+    y = df[target] if problem_type != "Clustering" else None
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) if problem_type != "Clustering" else (X, X.copy(), None, None)
+
+    # FIXED FEATURE SELECTION
+    numerical_features = X.select_dtypes(include=np.number).columns.tolist()
+    categorical_features = X.select_dtypes(exclude=np.number).columns.tolist()
+    X_train_processed, X_test_processed, feature_names, _ = preprocess_data(X_train, X_test, numerical_features, categorical_features)
+
+    if problem_type in ["Binary Classification", "Multi-Class"] and y is not None:
+        y_train = le.transform(y_train) if le else y_train
+        y_test = le.transform(y_test) if le else y_test
+        if problem_type == "Multi-Class":
+            y_train = tf.keras.utils.to_categorical(y_train)
+            y_test = tf.keras.utils.to_categorical(y_test)
+
+    # Validation
+    st.subheader("Model Validation")
+    metrics, y_pred = evaluate_model(model, X_test_processed, y_test, problem_type, le)
+    col1, col2 = st.columns(2)
+    with col1:
+        for metric, value in metrics.items():
+            st.metric(metric, f"{value:.4f}" if isinstance(value, float) else value)
+
+    with col2:
+        if problem_type == "Regression":
+            fig = px.scatter(x=y_test, y=y_pred, labels={"x": "Actual", "y": "Predicted"}, title="Predicted vs Actual")
+            st.plotly_chart(fig)
+        elif problem_type == "Binary Classification":
+            y_pred_proba = model.predict_proba(X_test_processed)[:, 1] if hasattr(model, 'predict_proba') else y_pred # Handle cases where predict_proba isn't available
+            fpr, tpr, _ = roc_curve(y_test, y_pred_proba)
+            roc_auc = auc(fpr, tpr)
+            fig = px.area(x=fpr, y=tpr, title=f"ROC Curve (AUC = {roc_auc:.2f})",
+                         labels={"x": "False Positive Rate", "y": "True Positive Rate"})
+            st.plotly_chart(fig)
+        elif problem_type == "Multi-Class":
+            y_pred_classes = np.argmax(model.predict(X_test_processed), axis=1) if isinstance(model, keras.Model) else model.predict(X_test_processed)
+            y_test_classes = np.argmax(y_test, axis=1)
+            cm = np.zeros((y_train.shape[1], y_train.shape[1]))
+            for i, j in zip(y_test_classes, y_pred_classes):
+                cm[i, j] += 1
+            fig = px.imshow(cm, title="Confusion Matrix", labels={"x": "Predicted", "y": "Actual"})
+            st.plotly_chart(fig)
+
+            # Display Classification Report
+            report = classification_report(y_test_classes, y_pred_classes, target_names=le.classes_ if le else [str(i) for i in range(y_train.shape[1])], zero_division=0)
+            st.text("Classification Report:\n" + report)
+        elif problem_type == "Clustering":
+            labels = y_pred
+            fig = px.scatter(x=X_test_processed[:, 0], y=X_test_processed[:, 1], color=labels, title="Cluster Visualization")
+            st.plotly_chart(fig)
+
+    # Dimensionality Reduction
+    st.subheader("Dimensionality Reduction")
+    method = st.selectbox("Method", ["PCA", "SVD", "t-SNE", "UMAP"])
+    n_components = st.slider("Components", 2, min(X_train_processed.shape[1], 10), 2)
+
+    if method == "PCA":
+        reducer = PCA(n_components=n_components)
+        X_reduced = reducer.fit_transform(X_train_processed)
+        fig = px.bar(x=range(n_components), y=reducer.explained_variance_ratio_, title="Explained Variance Ratio")
+        st.plotly_chart(fig)
+    elif method == "SVD":
+        reducer = TruncatedSVD(n_components=n_components)
+        X_reduced = reducer.fit_transform(X_train_processed)
+        fig = px.bar(x=range(n_components), y=reducer.explained_variance_ratio_, title="Explained Variance Ratio")
+        st.plotly_chart(fig)
+    elif method == "t-SNE":
+        X_reduced = TSNE(n_components=n_components, random_state=42).fit_transform(X_train_processed)
+    elif method == "UMAP":
+        X_reduced = umap.UMAP(n_components=n_components, random_state=42).fit_transform(X_train_processed)
+
+    if n_components >= 2:
+        fig = px.scatter(x=X_reduced[:, 0], y=X_reduced[:, 1], color=y_train if problem_type != "Clustering" else y_pred,
+                        title=f"{method} Visualization")
+        st.plotly_chart(fig)
+
+    # Interpretability
+    st.subheader("Interpretability")
+    try:
+        explainer = shap.KernelExplainer(model.predict, X_test_processed[:50]) if isinstance(model, keras.Model) else shap.Explainer(model, X_test_processed)
+        shap_values = explainer.shap_values(X_test_processed[:50])
+
+        if problem_type == "Regression":
+            shap_fig, ax = plt.subplots()
+            shap.summary_plot(shap_values, X_test_processed[:50], feature_names=feature_names, show=False)
+            st.pyplot(shap_fig)
+        elif problem_type in ["Binary Classification", "Multi-Class"]:
+            class_names = le.classes_ if le else [str(i) for i in range(y_train.shape[1])]
+            for i in range(len(class_names)):
+                shap_fig, ax = plt.subplots()
+                shap.summary_plot(shap_values[i], X_test_processed[:50], feature_names=feature_names, class_names=class_names, show=False)
+                st.pyplot(shap_fig)
+        else:
+            st.write("SHAP plots are not directly applicable to Clustering problems.")
+    except Exception as e:
+        st.error(f"Error generating SHAP plot: {e}")
+
+# Custom CSS
+st.markdown("""
+    <style>
+    .stButton>button {background-color: #4CAF50; color: white;}
+    h1, h2 {color: #1e3a8a;}
+    </style>
+""", unsafe_allow_html=True)