Enlarged the scope of the analytics section

app.py

@@ -34,10 +34,10 @@ home = st.Page("sections/home.py", title="🏠 Home")
 upload = st.Page("sections/upload.py", title="📥 Upload")
 statistics = st.Page("sections/statistics.py", title="📈 Statistics")
 analyze = st.Page("sections/analyze.py", title="🔍 Analyze")
-ml = st.Page("sections/ml.py", title="🤖 Machine Learning")
+analytics = st.Page("sections/analytics.py", title="🤖 Analytics")
 alerts = st.Page("sections/alerts.py", title="🚨 Alerts")
 about = st.Page("sections/about.py", title="📄 About")
 test_filtre = st.Page("sections/test_filtre.py", title="test_filtre")
 
-pg = st.navigation([home, upload, statistics, analyze, ml, alerts, test_filtre])
+pg = st.navigation([home, upload, statistics, analyze, analytics, alerts, test_filtre])
 pg.run()

sections/{ml.py → analytics.py}

@@ -4,14 +4,14 @@ import streamlit as st
 import polars as pl
 
 from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import OneHotEncoder
 from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
-import matplotlib.pyplot as plt
 
 if "parsed_df" not in st.session_state:
     st.session_state.parsed_df = None
 
 # Page title
-st.title("Machine Learning")
+st.title("Analytiques")
 
 # Loading data
 if st.session_state.parsed_df is None:
@@ -19,18 +19,36 @@ if st.session_state.parsed_df is None:
     st.stop()
 
 data = st.session_state.parsed_df
+data = data.select(["portdest","protocole","regle1","status"])
+
+# Sélectionner toutes les colonnes numériques
+quanti = data.select(pl.col(pl.String))
+
+# Sélectionner toutes les colonnes de type chaîne
+quali = data.select(pl.col(pl.Int64))
 
 ##############################################
 ####            Preprocessing             ####
 ##############################################
 
-# Normalisation des données (Standardisation : moyenne = 0, écart-type = 1)
+# Normalisation des données quanti (Standardisation : moyenne = 0, écart-type = 1)
+
 scaler = StandardScaler()
-df_scaled = scaler.fit_transform(data.to_pandas())
+data_quanti = scaler.fit_transform(quanti.to_pandas())
+
+# Convertir de nouveau en DataFrame Polars
+data_quanti = pl.from_pandas(pd.DataFrame(data_quanti, columns=data_quanti.columns))
+
+# Encodage one-hot des données quali
+
+encoder = OneHotEncoder()
+data_quali = encoder.fit_transform(quali.to_pandas())
 
 # Convertir de nouveau en DataFrame Polars
-df_scaled = pl.from_pandas(pd.DataFrame(df_scaled, columns=data.columns))
+data_quali = pl.from_pandas(pd.DataFrame(data_quali, columns=data_quali.columns))
+
 
+df = pl.concat([data_quanti, data_quali], how="diagonal")
 
 ###############################################
 ####              Clustering               ####
@@ -43,15 +61,15 @@ if st.button("Start clustering"):
                 # Appliquer K-Means avec k optimal choisi
                 k_optimal = 2  # Par exemple, supposons que k = 3
                 kmeans = KMeans(n_clusters=k_optimal, random_state=42)
-                df_scaled = df_scaled.with_columns(pl.Series(kmeans.fit_predict(df_scaled.to_pandas()), name='cluster_kmeans'))
+                df = df.with_columns(pl.Series(kmeans.fit_predict(df.to_pandas()), name='cluster_kmeans'))
 
                 # Appliquer DBSCAN (epsilon et min_samples sont des hyperparamètres)
                 # dbscan = DBSCAN(eps=0.5, min_samples=10)
-                # df_scaled = df_scaled.with_columns(pl.Series(dbscan.fit_predict(df_scaled.to_pandas()), name='cluster_dbscan'))
+                # df = df.with_columns(pl.Series(dbscan.fit_predict(df.to_pandas()), name='cluster_dbscan'))
 
                 # Appliquer Agglomerative Clustering
                 # agg_clustering = AgglomerativeClustering(n_clusters=2)
-                # df_scaled = df_scaled.with_columns(pl.Series(agg_clustering.fit_predict(df_scaled.to_pandas()), name='cluster_agg'))
+                # df = df.with_columns(pl.Series(agg_clustering.fit_predict(df.to_pandas()), name='cluster_agg'))
 
                 ###############################################################
                 ####              Visualisation des clusters               ####
@@ -62,12 +80,12 @@ if st.button("Start clustering"):
                 from sklearn.decomposition import PCA
 
                 pca = PCA(n_components=2)
-                df_pca = pca.fit_transform(df_scaled.to_pandas())
+                df_pca = pca.fit_transform(df.to_pandas())
 
                 fig = px.scatter(
                     x=df_pca[:, 0],
                     y=df_pca[:, 1],
-                    color=df_scaled['cluster_kmeans'],
+                    color=df['cluster_kmeans'],
                     color_continuous_scale='viridis',
                     title='Clustering coupled with PCA',
                     labels={'x': 'Component 1', 'y': 'Component 2', 'color': 'Cluster'},