Update app.py

app.py

@@ -1,5 +1,6 @@
 import streamlit as st
 import pandas as pd
+import dask.dataframe as dd
 from sklearn.model_selection import train_test_split
 from sklearn.preprocessing import LabelEncoder
 from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
@@ -15,15 +16,18 @@ import seaborn as sns
 from io import BytesIO
 
 # File uploader
-st.title("Model Training with Metrics and Correlation Heatmap")
+st.title("Model Training with Metrics and Correlation Heatmap (Optimized for Large Datasets)")
 uploaded_file = st.file_uploader("Choose a CSV file", type=["csv"])
 
 if uploaded_file is not None:
-    df = pd.read_csv(uploaded_file)
-    
-    # Show the dataset
-    st.write("Dataset:")
-    st.dataframe(df)
+    # Use Dask for large datasets
+    st.write("Loading dataset...")
+    df = dd.read_csv(uploaded_file).compute()  # Convert Dask DataFrame to Pandas DataFrame for further processing
+    st.success("Dataset loaded successfully!")
+
+    # Show a preview of the dataset
+    st.write("Dataset Preview:")
+    st.dataframe(df.head(100))  # Display only the first 100 rows for better performance
 
     # Convert categorical (str) data to numerical
     st.write("Converting Categorical Columns to Numerical Values:")
@@ -32,12 +36,12 @@ if uploaded_file is not None:
     for col in df.columns:
         if df[col].dtype == 'object' or len(df[col].unique()) <= 10:
             st.write(f"Encoding Column: **{col}**")
-            df[col] = label_encoder.fit_transform(df[col])
+            df[col] = label_encoder.fit_transform(df[col].astype(str))
     
     # Display the dataset after conversion
-    st.write("Dataset After Conversion:")
-    st.dataframe(df)
-    
+    st.write("Dataset After Conversion (Preview):")
+    st.dataframe(df.head(100))  # Display a preview of the converted dataset
+
     # Handle Null Values (Missing Data)
     st.write("Handling Missing (Null) Values:")
     fill_method = st.selectbox("Choose how to handle missing values", ["Drop rows", "Fill with mean/median"])
@@ -72,8 +76,8 @@ if uploaded_file is not None:
     df = cap_extreme_values(df)
     
     # Show cleaned dataset
-    st.write("Cleaned Dataset:")
-    st.dataframe(df)
+    st.write("Cleaned Dataset (Preview):")
+    st.dataframe(df.head(100))  # Display a preview of the cleaned dataset
 
     # Add clean data download option
     st.subheader("Download Cleaned Dataset")
@@ -107,7 +111,7 @@ if uploaded_file is not None:
     high_corr = corr.abs().unstack().sort_values(ascending=False).drop_duplicates()
     high_corr = high_corr[high_corr.index.get_level_values(0) != high_corr.index.get_level_values(1)]
     high_corr_df = pd.DataFrame(high_corr)
-    st.write(high_corr_df)
+    st.write(high_corr_df.head(10))  # Show top 10 highly correlated pairs
 
     target = st.selectbox("Select Target Variable", df.columns)
     features = [col for col in df.columns if col != target]
@@ -144,44 +148,6 @@ if uploaded_file is not None:
         st.subheader("Classification Model Performance Metrics")
         st.dataframe(metrics_df)
 
-        # Save metrics as PNG (table form)
-        fig, ax = plt.subplots(figsize=(8, 4))
-        ax.axis('tight')
-        ax.axis('off')
-        table = plt.table(cellText=metrics_df.values, colLabels=metrics_df.columns, cellLoc='center', loc='center')
-        table.auto_set_font_size(False)
-        table.set_fontsize(10)
-        table.auto_set_column_width(col=list(range(len(metrics_df.columns))))
-        buf = BytesIO()
-        fig.savefig(buf, format="png")
-        buf.seek(0)
-        st.download_button(
-            label="Download Classification Metrics Table as PNG",
-            data=buf,
-            file_name="classification_metrics_table.png",
-            mime="image/png"
-        )
-        
-        # Visualization (Bar Graphs for Classification)
-        st.subheader("Classification Model Performance Metrics Graph")
-        metrics_df.set_index('Model', inplace=True)
-        ax = metrics_df.plot(kind='bar', figsize=(10, 6), colormap='coolwarm', rot=45)
-        plt.title("Classification Models - Performance Metrics")
-        plt.ylabel("Scores")
-        plt.xlabel("Models")
-        st.pyplot(plt)
-
-        # Download button for the bar graph
-        buf = BytesIO()
-        ax.figure.savefig(buf, format="png")
-        buf.seek(0)
-        st.download_button(
-            label="Download Classification Performance Graph as PNG",
-            data=buf,
-            file_name="classification_performance_graph.png",
-            mime="image/png"
-        )
-
     else:  # Continuous target (regression)
         st.subheader("Regression Model Training")
         regressors = {
@@ -209,41 +175,3 @@ if uploaded_file is not None:
         regression_metrics_df = pd.DataFrame(regression_metrics)
         st.subheader("Regression Model Performance Metrics")
         st.dataframe(regression_metrics_df)
-
-        # Save metrics as PNG (table form)
-        fig, ax = plt.subplots(figsize=(8, 4))
-        ax.axis('tight')
-        ax.axis('off')
-        table = plt.table(cellText=regression_metrics_df.values, colLabels=regression_metrics_df.columns, cellLoc='center', loc='center')
-        table.auto_set_font_size(False)
-        table.set_fontsize(10)
-        table.auto_set_column_width(col=list(range(len(regression_metrics_df.columns))))
-        buf = BytesIO()
-        fig.savefig(buf, format="png")
-        buf.seek(0)
-        st.download_button(
-            label="Download Regression Metrics Table as PNG",
-            data=buf,
-            file_name="regression_metrics_table.png",
-            mime="image/png"
-        )
-
-        # Visualization (Bar Graphs for Regression)
-        st.subheader("Regression Model Performance Metrics Graph")
-        regression_metrics_df.set_index('Model', inplace=True)
-        regression_metrics_df.plot(kind='bar', figsize=(10, 6), colormap='coolwarm', rot=45)
-        plt.title("Regression Models - Performance Metrics")
-        plt.ylabel("Scores")
-        plt.xlabel("Models")
-        st.pyplot(plt)
-
-        # Download button for the bar graph
-        buf = BytesIO()
-        plt.savefig(buf, format="png")
-        buf.seek(0)
-        st.download_button(
-            label="Download Regression Performance Graph as PNG",
-            data=buf,
-            file_name="regression_performance_graph.png",
-            mime="image/png"
-        )