Changed the clustering approach

sections/analytics.py

@@ -24,39 +24,23 @@ if st.session_state.parsed_df is None:
 data = st.session_state.parsed_df
 data = data.select(["portdst","protocole","rule","action"])
 
-# Sélectionner toutes les colonnes numériques
-quanti = data.select(pl.col(pl.Int64))
-
-# Sélectionner toutes les colonnes de type chaîne
-quali = data.select(pl.col(pl.String))
-
 ##############################################
 ####            Preprocessing             ####
 ##############################################
 
-# Normalisation des données quanti (Standardisation : moyenne = 0, écart-type = 1)
-
-scaler = StandardScaler()
-data_quanti = scaler.fit_transform(quanti.to_pandas())
-
-# Convertir de nouveau en DataFrame Polars
-data_quanti = pl.from_pandas(pd.DataFrame(data_quanti, columns=quanti.columns))
-
-# Encodage one-hot des données quali
+# Encodage one-hot
 
 encoder = OneHotEncoder(sparse_output=False)
-data_quali = encoder.fit_transform(quali.to_pandas())
+data_encoded = encoder.fit_transform(data.to_pandas())
 
 col_names = [
     f"{feature}_{category}" 
-    for feature, categories in zip(quali.columns, encoder.categories_)
+    for feature, categories in zip(data.columns, encoder.categories_)
     for category in categories
 ]
 
 # Convertir de nouveau en DataFrame Polars
-data_quali = pl.from_pandas(pd.DataFrame(data_quali, columns=col_names))
-
-df = pl.concat([data_quanti, data_quali], how="horizontal")
+data_encoded = pl.from_pandas(pd.DataFrame(data_encoded, columns=col_names))
 
 ###############################################
 ####              Clustering               ####
@@ -66,11 +50,16 @@ if st.button("Start clustering"):
     if st.session_state.parsed_df is not None:
         with st.spinner("Searching the clusters..."):
             try:
+
+                pca = PCA(n_components=2)
+                df_pca = pca.fit_transform(data_encoded.to_pandas())
+
                 # Appliquer K-Means avec k optimal choisi
                 k_optimal = 2  # Par exemple, supposons que k = 3
                 kmeans = KMeans(n_clusters=k_optimal, random_state=42)
-                preds = kmeans.fit_predict(df.to_pandas())
-                df_clust = df.with_columns(pl.Series(values=preds, name='cluster_kmeans'))
+                preds = kmeans.fit_predict(df_pca)
+                df_pca = pl.from_pandas(pd.DataFrame(df_pca, columns=[f"Component {i+1}" for i in range(k_optimal)]))
+                df_clust = df_pca.with_columns(pl.Series(values=preds, name='cluster_kmeans'))
 
                 if df_clust.shape[0] > 200000: # 200k
                     perc = 200000/df_clust.shape[0]
@@ -88,14 +77,14 @@ if st.button("Start clustering"):
 
                 # Visualisation des clusters (en 2D avec PCA)                
 
-                pca = PCA(n_components=2)
-                df_pca = pca.fit_transform(df_ech.to_pandas())
+                # pca = PCA(n_components=2)
+                # df_pca = pca.fit_transform(df_ech.to_pandas())
 
                 fig = px.scatter(
-                    x=df_pca[:, 0],
-                    y=df_pca[:, 1],
-                    color=df_ech['cluster_kmeans'],
-                    color_continuous_scale='viridis',
+                    x=df_ech.select("Component 1").to_numpy().flatten(),
+                    y=df_ech.select("Component 2").to_numpy().flatten(),
+                    color=df_ech.select('cluster_kmeans').to_numpy().flatten().astype(str),
+                    color_discrete_map={"0": "rebeccapurple", "1": "gold"},
                     title='Clustering coupled with PCA',
                     labels={'x': 'Component 1', 'y': 'Component 2', 'color': 'Cluster'},
                 )
@@ -104,21 +93,20 @@ if st.button("Start clustering"):
                     xaxis_title='Component 1',
                     yaxis_title='Component 2'
                 )
-
                 # fig.show()
                 st.plotly_chart(fig, use_container_width=True)
                 
             except Exception as e:
-                st.error(f"An error occured while doing the clustering : {e}")
+                st.error(f"An error occured while doing the clustering : {e.with_traceback(None)}")
 
         with st.spinner("Performing some more data analysis..."):
             try:
-                data = data.with_columns(pl.Series(name="cluster_kmeans", values=df_clust.select("cluster_kmeans")))
+                data_clust = data.with_columns(pl.Series(name="cluster_kmeans", values=df_clust.select("cluster_kmeans")))
                 # Analyse des variables qualitatives par cluster
-                for col in quali.columns: # protocole, action
+                for col in data.columns : # portdst, protocole, rule, action
                     fig = make_subplots(rows=1, cols=2)
 
-                    data_filtered = data.filter(pl.col("cluster_kmeans") == 0)
+                    data_filtered = data_clust.filter(pl.col("cluster_kmeans") == 0)
                     freq_df = data_filtered.group_by(col).agg(pl.count(col).alias("frequency"))
 
                     fig.add_trace(
@@ -127,7 +115,7 @@ if st.button("Start clustering"):
                         row=1, col=1
                     )
 
-                    data_filtered = data.filter(pl.col("cluster_kmeans") == 1)
+                    data_filtered = data_clust.filter(pl.col("cluster_kmeans") == 1)
                     freq_df = data_filtered.group_by(col).agg(pl.count(col).alias("frequency"))
 
                     fig.add_trace(
@@ -144,32 +132,32 @@ if st.button("Start clustering"):
                     )
                     st.plotly_chart(fig, use_container_width=True)
 
-                # Analyse de la variable quantitative par cluster
-                for col in quanti.columns: # protocole, rule, action
-                    fig = make_subplots(rows=1, cols=2)
+                # # Analyse de la variable quantitative par cluster
+                # for col in quanti.columns: # protocole, rule, action
+                #     fig = make_subplots(rows=1, cols=2)
 
-                    data_filtered = data.filter(pl.col("cluster_kmeans") == 0)
+                #     data_filtered = data.filter(pl.col("cluster_kmeans") == 0)
 
-                    # Ajouter le premier histogramme
-                    fig.add_trace(
-                        go.Histogram(x=data_filtered[col], name="Cluster 0", marker_color="rebeccapurple"),
-                        row=1, col=1
-                    )
+                #     # Ajouter le premier histogramme
+                #     fig.add_trace(
+                #         go.Histogram(x=data_filtered[col], name="Cluster 0", marker_color="rebeccapurple"),
+                #         row=1, col=1
+                #     )
 
-                    data_filtered = data.filter(pl.col("cluster_kmeans") == 1)
+                #     data_filtered = data.filter(pl.col("cluster_kmeans") == 1)
 
-                    # Ajouter le deuxième histogramme
-                    fig.add_trace(
-                        go.Histogram(x=data_filtered[col], name="Cluster 1", marker_color="gold"),
-                        row=1, col=2
-                    )
+                #     # Ajouter le deuxième histogramme
+                #     fig.add_trace(
+                #         go.Histogram(x=data_filtered[col], name="Cluster 1", marker_color="gold"),
+                #         row=1, col=2
+                #     )
 
-                    # Mettre à jour la mise en page pour améliorer l'apparence
-                    fig.update_layout(
-                        title_text=f"Histograms of {col}",
-                        showlegend=True,
-                    )
-                    st.plotly_chart(fig, use_container_width=True)
+                #     # Mettre à jour la mise en page pour améliorer l'apparence
+                #     fig.update_layout(
+                #         title_text=f"Histograms of {col}",
+                #         showlegend=True,
+                #     )
+                #     st.plotly_chart(fig, use_container_width=True)
 
             except Exception as e:
                 st.error(f"An error occured while doing the data analysis : {e}")

sections/analyze.py

@@ -100,8 +100,8 @@ with tab1:
             st.markdown("### 🔢 Port")
             if "portdst" in data.columns:
                 min_port, max_port = (
-                    int(data["portdst"].min()),
-                    int(data["portdst"].max()),
+                    int(data["portdst"].cast(pl.Utf8).min()),
+                    int(data["portdst"].cast(pl.Utf8).max()),
                 )
 
                 # Initialize port range in session state if not present
@@ -272,7 +272,7 @@ with tab3:
         " (portdst < 1024 and action == 'PERMIT')"
     )
     top_ports = (
-        data.filter((pl.col("portdst") < 1024) & (pl.col("action") == "PERMIT"))
+        data.filter((pl.col("portdst").cast(pl.Int64) < 1024) & (pl.col("action") == "PERMIT"))
         .group_by("portdst")
         .agg(pl.count("portdst").alias("count"))
         .sort("count", descending=True)
@@ -483,8 +483,8 @@ with tab5:
 
     # 🔹 Sankey entre IP source et port destination
     df = data_filtered.with_columns(
-        data_filtered["portdst"].cast(pl.Utf8)
-    )  # Convertir les ports en chaînes pour éviter les erreurs
+        data_filtered["portdst"]
+    ) 
     create_sankey(df, "ipsrc", "portdst")
 
     st.subheader("Connections where access were identified as : DENY")
@@ -495,6 +495,6 @@ with tab5:
 
     # 🔹 Sankey entre IP source et port destination
     df = data_filtered.with_columns(
-        data_filtered["portdst"].cast(pl.Utf8)
-    )  # Convertir les ports en chaînes pour éviter les erreurs
+        data_filtered["portdst"]
+    )
     create_sankey(df, "ipsrc", "portdst")

sections/upload.py

@@ -79,9 +79,9 @@ if uploaded_file is not None:
                     "ipsrc": pl.Utf8,
                     "ipdst": pl.Utf8,
                     "protocole": pl.Utf8,
-                    "portsrc": pl.Int64,
-                    "portdst": pl.Int64,
-                    "rule": pl.Int64,
+                    "portsrc": pl.Utf8,
+                    "portdst": pl.Utf8,
+                    "rule": pl.Utf8,
                     "action": pl.Utf8,
                     "interface": pl.Utf8,
                     "unknown": pl.Utf8,