Update app.py

app.py

@@ -1,6 +1,7 @@
 import os
 import pandas as pd
 import numpy as np
+import seaborn as sns
 import matplotlib.pyplot as plt
 import matplotlib as mpl
 import pycaret
@@ -21,8 +22,8 @@ hide_streamlit_style = """
 st.markdown(hide_streamlit_style, unsafe_allow_html=True)
 
 with st.sidebar:
-    image = Image.open('./itaca_logo.png')
-    st.image(image,use_column_width=True)
+    image = Image.open('itaca_logo.png')
+    st.image(image, width=150) #,use_column_width=True)
     page = option_menu(menu_title='Menu',
                        menu_icon="robot",
                        options=["Clustering Analysis",
@@ -32,6 +33,24 @@ with st.sidebar:
                        default_index=0
                        )
 
+    # Additional section below the option menu
+    # st.markdown("---")  # Add a separator line
+    st.header("Settings")
+    
+   # Define the options for the dropdown list
+    numclusters = [2, 3, 4, 5, 6]
+    # selected_clusters = st.selectbox("Choose a number of clusters", numclusters)
+    selected_clusters = st.slider("Choose a number of clusters", min_value=2, max_value=10, value=4)
+    
+    p_remove_multicollinearity = st.checkbox("Remove Multicollinearity", value=False)
+    p_multicollinearity_threshold = st.slider("Choose multicollinearity thresholds", min_value=0.0, max_value=1.0, value=0.9)
+    # p_remove_outliers = st.checkbox("Remove Outliers", value=False)
+    # p_outliers_method = st.selectbox ("Choose an Outlier Method", ["iforest", "ee", "lof"])
+    p_transformation = st.checkbox("Choose Power Transform", value = False)
+    p_normalize = st.checkbox("Choose Normalize", value = False)
+    p_pca = st.checkbox("Choose PCA", value = False)
+    p_pca_method = st.selectbox ("Choose a PCA Method", ["linear", "kernel", "incremental"])
+
 st.title('ITACA Insurance Core AI Module')
 
 if page == "Clustering Analysis":
@@ -52,10 +71,9 @@ if page == "Clustering Analysis":
     all_files = os.listdir(directory)
     # Filter files to only include CSV files
     csv_files = [file for file in all_files if file.endswith(".csv")]
-
     # Select a CSV file from the list
     selected_csv = st.selectbox("Select a CSV file from the list", ["None"] + csv_files)
-        
+
     # Upload the CSV file
     uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
     
@@ -63,11 +81,6 @@ if page == "Clustering Analysis":
     clusteringmodel = ['kmeans', 'ap', 'meanshift', 'sc', 'hclust', 'dbscan', 'optics', 'birch']
     selected_model = st.selectbox("Choose a clustering model", clusteringmodel)
 
-    # Define the options for the dropdown list
-    numclusters = [2, 3, 4, 5, 6]
-    # selected_clusters = st.selectbox("Choose a number of clusters", numclusters)
-    selected_clusters = st.slider("Choose a number of clusters", min_value=2, max_value=10, value=4)
-
     # Read and display the CSV file
     if selected_csv != "None" or uploaded_file is not None:
         if uploaded_file:
@@ -80,23 +93,51 @@ if page == "Clustering Analysis":
         else:
             insurance_claims = pd.read_csv(selected_csv)
 
-        s = setup(insurance_claims, session_id = 123)
+        insurance_claims.describe().T
 
-        exp_clustering = ClusteringExperiment()
+        cat_col = insurance_claims.select_dtypes(include=['object']).columns
+        num_col = insurance_claims.select_dtypes(exclude=['object']).columns
 
-        # init setup on exp
-        exp_clustering.setup(insurance_claims, session_id = 123)
+        # insurance_claims[num_col].hist(bins=15, figsize=(20, 15), layout=(5, 4))
+        # Calculate the correlation matrix
+        corr_matrix = insurance_claims[num_col].corr()
+        # Create a Matplotlib figure
+        fig, ax = plt.subplots(figsize=(12, 8))
+        # Create a heatmap using seaborn
+        sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', ax=ax)
+        # Set the title for the heatmap
+        ax.set_title('Correlation Heatmap')
+        # Display the heatmap in Streamlit
+        st.pyplot(fig)
+
+        all_columns = insurance_claims.columns.tolist()
+        selected_columns = st.multiselect("Choose columns", all_columns, default=all_columns)
 
         if st.button("Prediction"):
+            insurance_claims = insurance_claims[selected_columns].copy()
+            
+            s = setup(insurance_claims, session_id = 123, remove_multicollinearity=p_remove_multicollinearity, multicollinearity_threshold=p_multicollinearity_threshold,
+                    # remove_outliers=p_remove_outliers, outliers_method=p_outliers_method, 
+                    transformation=p_transformation, 
+                    normalize=p_normalize, pca=p_pca, pca_method=p_pca_method)
+            exp_clustering = ClusteringExperiment()
+            # init setup on exp
+            exp_clustering.setup(insurance_claims, session_id = 123)
+
             with st.spinner("Analyzing..."):
                 # train kmeans model
                 cluster_model = create_model(selected_model, num_clusters = selected_clusters)
 
                 cluster_model_2 = assign_model(cluster_model)
+                # Calculate summary statistics for each cluster
+                cluster_summary = cluster_model_2.groupby('Cluster').agg(['count', 'mean', 'median', 'min', 'max', 
+                                                                             'std', 'var', 'sum', ('quantile_25', lambda x: x.quantile(0.25)), 
+                                                                             ('quantile_75', lambda x: x.quantile(0.75)), 'skew'])
+                cluster_summary
                 cluster_model_2
 
-                all_metrics = get_metrics()
-                all_metrics
+                # all_metrics = get_metrics()
+                # all_metrics
 
                 cluster_results = pull()
                 cluster_results
@@ -137,7 +178,6 @@ elif page == "Anomaly Detection":
     all_files = os.listdir(directory)
     # Filter files to only include CSV files
     csv_files = [file for file in all_files if file.endswith(".csv")]
-
     # Select a CSV file from the list
     selected_csv = st.selectbox("Select a CSV file from the list", ["None"] + csv_files)
     
@@ -159,15 +199,24 @@ elif page == "Anomaly Detection":
                 insurance_claims = pd.read_csv (uploaded_file, sep=delimiter, encoding='latin-1')
         else:
             insurance_claims = pd.read_csv(selected_csv)
-        
-        s = setup(insurance_claims, session_id = 123)
-
-        exp_anomaly = AnomalyExperiment()
 
-        # init setup on exp
-        exp_anomaly.setup(insurance_claims, session_id = 123)
+        all_columns = insurance_claims.columns.tolist()
+        selected_columns = st.multiselect("Choose columns", all_columns, default=all_columns)
 
         if st.button("Prediction"):
+            insurance_claims = insurance_claims[selected_columns].copy()
+            
+            # s = setup(insurance_claims, session_id = 123)
+
+            s = setup(insurance_claims, session_id = 123, remove_multicollinearity=p_remove_multicollinearity, multicollinearity_threshold=p_multicollinearity_threshold,
+                        # remove_outliers=p_remove_outliers, outliers_method=p_outliers_method, 
+                        transformation=p_transformation, 
+                        normalize=p_normalize, pca=p_pca, pca_method=p_pca_method)
+
+            exp_anomaly = AnomalyExperiment()
+            # init setup on exp
+            exp_anomaly.setup(insurance_claims, session_id = 123)
+        
             with st.spinner("Analyzing..."):
                 # train model
                 anomaly_model = create_model(selected_model)
@@ -180,5 +229,4 @@ elif page == "Anomaly Detection":
 
                 # plot
                 plot_model(anomaly_model, plot = 'tsne', display_format = 'streamlit')
-                plot_model(anomaly_model, plot = 'umap', display_format = 'streamlit')
-
+                plot_model(anomaly_model, plot = 'umap', display_format = 'streamlit')