Added Sankey

sections/analytics.py

@@ -7,7 +7,8 @@ import polars as pl
 
 from sklearn.preprocessing import StandardScaler
 from sklearn.preprocessing import OneHotEncoder
-from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
+from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
 
 if "parsed_df" not in st.session_state:
     st.session_state.parsed_df = None
@@ -21,7 +22,7 @@ if st.session_state.parsed_df is None:
     st.stop()
 
 data = st.session_state.parsed_df
-data = data.select(["portdest","protocole","regle1","status"])
+data = data.select(["portdst","protocole","regle","action"])
 
 # Sélectionner toutes les colonnes numériques
 quanti = data.select(pl.col(pl.Int64))
@@ -75,21 +76,13 @@ if st.button("Start clustering"):
                                     .groupby("cluster_kmeans", group_keys=False)
                                     .apply(lambda x: x.sample(frac=0.05, random_state=42))
                                     )
-                # dbscan = DBSCAN(eps=0.5, min_samples=10)
-                # preds = dbscan.fit_predict(df.to_pandas())
-                # df = df.with_columns(pl.Series(values=preds, name='cluster_dbscan'))
-
-                # agg_clustering = AgglomerativeClustering(n_clusters=2)
-                # preds = agg_clustering.fit_predict(df.to_pandas())
-                # df = df.with_columns(pl.Series(values=preds, name='cluster_agg'))                
 
                 ###############################################################
                 ####              Visualisation des clusters               ####
                 ###############################################################
 
 
-                # Visualisation des clusters (en 2D avec PCA)
-                from sklearn.decomposition import PCA
+                # Visualisation des clusters (en 2D avec PCA)                
 
                 pca = PCA(n_components=2)
                 df_pca = pca.fit_transform(df_ech.to_pandas())
@@ -117,27 +110,8 @@ if st.button("Start clustering"):
         with st.spinner("Performing some more data analysis..."):
             try:
                 data = data.with_columns(pl.Series(name="cluster_kmeans", values=df_clust.select("cluster_kmeans")))
-                cols = ["protocole","regle1","status"]
-                for col in cols:                                   
-                    # fig = px.bar(freq_df, x=col, y='frequency',                                 
-                    #             title=f'{col} frequency',
-                    #             labels={'categorie': 'Category', 'frequence': 'Frequency'},
-                    #             color=col)
-                    # fig.update_layout(xaxis_title='Categories', yaxis_title='Frequency')
-                    # st.plotly_chart(fig, use_container_width=True)
-
-                    # data_filtered = data.filter(pl.col("cluster_kmeans") == 0)
-                    # freq_df = data_filtered.group_by(col).agg(pl.count(col).alias("frequency"))
-                    
-                    # fig = px.bar(freq_df, x=col, y='frequency',                                 
-                    #             title=f'{col} frequency',
-                    #             labels={'categorie': 'Category', 'frequence': 'Frequency'},
-                    #             color=col)
-                    # fig.update_layout(xaxis_title='Categories', yaxis_title='Frequency')
-                    # st.plotly_chart(fig, use_container_width=True)
-
-
-
+                # Analyse des variables qualitatives par cluster
+                for col in quali.columns: # protocole, regle, action
                     fig = make_subplots(rows=1, cols=2)
 
                     data_filtered = data.filter(pl.col("cluster_kmeans") == 0)
@@ -166,13 +140,15 @@ if st.button("Start clustering"):
                     )
                     st.plotly_chart(fig, use_container_width=True)
 
+                # Analyse de la variable quantitative par cluster
+
                 fig = make_subplots(rows=1, cols=2)
 
                 data_filtered = data.filter(pl.col("cluster_kmeans") == 0)
 
                 # Ajouter le premier histogramme
                 fig.add_trace(
-                    go.Histogram(x=data_filtered["portdest"], name="Cluster 0", marker_color="rebeccapurple"),
+                    go.Histogram(x=data_filtered["portdst"], name="Cluster 0", marker_color="rebeccapurple"),
                     row=1, col=1
                 )
 
@@ -180,7 +156,7 @@ if st.button("Start clustering"):
 
                 # Ajouter le deuxième histogramme
                 fig.add_trace(
-                    go.Histogram(x=data_filtered["portdest"], name="Cluster 1", marker_color="gold"),
+                    go.Histogram(x=data_filtered["portdst"], name="Cluster 1", marker_color="gold"),
                     row=1, col=2
                 )
 
@@ -194,19 +170,4 @@ if st.button("Start clustering"):
             except Exception as e:
                 st.error(f"An error occured while doing the data analysis : {e}")
     else:
-        st.warning("Please parse the log file first.")
-
-# Choisir le nombre de clusters (méthode du coude)
-# inertia = []
-# for k in range(1, 11):
-#     kmeans = KMeans(n_clusters=k, random_state=42)
-#     kmeans.fit(df_scaled.to_pandas())
-#     inertia.append(kmeans.inertia_)
-
-# # Tracer la courbe pour la méthode du coude
-# plt.plot(range(1, 11), inertia, marker='o')
-# plt.title('Méthode du coude')
-# plt.xlabel('Nombre de clusters')
-# plt.ylabel('Inertie')
-# plt.show()
-
+        st.warning("Please parse the log file first.")

sections/analyze.py

@@ -2,6 +2,7 @@ import polars as pl
 import streamlit as st
 import ipaddress
 import plotly.express as px
+import plotly.graph_objs as go
 import pandas as pd
 
 if "parsed_df" not in st.session_state:
@@ -316,3 +317,38 @@ with tab3:
 # Onglet Sankey
 with tab4:
     st.subheader("Sankey Diagram")
+    
+    def create_sankey(df, source_col, target_col):
+        """ Crée un diagramme de Sankey entre deux colonnes """
+        df_grouped = df.groupby([source_col, target_col]).len().to_pandas()
+
+        # Création des nœuds
+        labels = list(pd.concat([df_grouped[source_col], df_grouped[target_col]]).unique())
+        label_to_index = {label: i for i, label in enumerate(labels)}
+
+        # Création des liens
+        sources = df_grouped[source_col].map(label_to_index)
+        targets = df_grouped[target_col].map(label_to_index)
+        values = df_grouped["len"]
+
+        # Création du Sankey Diagram
+        fig = go.Figure(go.Sankey(
+            node=dict(
+                pad=15, thickness=20, line=dict(color="black", width=0.5),
+                label=labels
+            ),
+            link=dict(
+                source=sources, target=targets, value=values
+            )
+        ))
+        
+        fig.update_layout(title_text=f"Flux entre {source_col} et {target_col}", font_size=10)
+        st.plotly_chart(fig, use_container_width=True)
+
+    # 🔹 Sankey entre IP source et IP destination
+    create_sankey(data, "ip_source", "ip_destination")
+
+    # 🔹 Sankey entre IP source et port destination
+    df = df.with_columns(df["port_destination"].cast(pl.Utf8))  # Convertir les ports en chaînes pour éviter les erreurs
+    create_sankey(data, "ip_source", "port_destination")
+