Update app.py

app.py

@@ -1,99 +1,134 @@
+import os
+import urllib.request
+import gzip
+import io
+
 import numpy as np
 import pandas as pd
+import networkx as nx
 from sklearn.cluster import KMeans
+
 import torch
 from torch import nn
 from torch.utils.data import TensorDataset, DataLoader
+
 import matplotlib.pyplot as plt
 import gradio as gr
 
 # -------------------------------------------------------
-# 1. Load dataset (Facebook Metrics of Cosmetic Brand)
+# 1. Download and load SNAP Facebook combined graph
 # -------------------------------------------------------
-DATA_PATH = "dataset_Facebook.csv"  # semicolon-separated file
-
-df = pd.read_csv(DATA_PATH, sep=";")
-df = df.fillna(0)
-
-# Rename important columns if needed
-df = df.rename(columns={
-    "Page total likes": "page_likes",
-    "Lifetime Post Total Impressions": "impressions",
-    "Lifetime Engaged Users": "engaged_users",
-    "comment": "comments",
-    "like": "likes",
-    "share": "shares"
-})
-
-# Fallback if some names are missing in your copy
-for col in ["comments", "likes", "shares"]:
-    if col not in df.columns:
-        df[col] = 0
+
+SNAP_URL = "https://snap.stanford.edu/data/facebook_combined.txt.gz"
+DATA_DIR = "data"
+os.makedirs(DATA_DIR, exist_ok=True)
+LOCAL_PATH = os.path.join(DATA_DIR, "facebook_combined.txt.gz")
+
+if not os.path.exists(LOCAL_PATH):
+    print("Downloading SNAP Facebook dataset...")
+    urllib.request.urlretrieve(SNAP_URL, LOCAL_PATH)
+else:
+    print("Using cached SNAP dataset.")
+
+print("Loading graph...")
+with gzip.open(LOCAL_PATH, "rt") as f:
+    G = nx.read_edgelist(f, nodetype=int)
+
+print(f"Graph loaded: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges")
+
+# Ensure largest connected component (should already be connected in this dataset)
+if not nx.is_connected(G):
+    largest_cc = max(nx.connected_components(G), key=len)
+    G = G.subgraph(largest_cc).copy()
+    print(f"After LCC: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges")
+
+nodes = list(G.nodes())
+node_index = {n: i for i, n in enumerate(nodes)}
+N = len(nodes)
 
 # -------------------------------------------------------
-# 2. Real behavioural features from dataset
+# 2. Real structural features from SNAP graph
 # -------------------------------------------------------
 
-engagement = df["comments"] + df["likes"] + df["shares"]
+# Degree
+deg = np.array([G.degree(n) for n in nodes], dtype=float)
 
-impressions = df["impressions"].replace(0, 1)
-interaction_rate = df["engaged_users"] / impressions
+# Clustering coefficient
+cc_dict = nx.clustering(G)
+cc = np.array([cc_dict[n] for n in nodes], dtype=float)
+
+# Average neighbor degree
+avg_nd_dict = nx.average_neighbor_degree(G)
+avg_nd = np.array([avg_nd_dict[n] for n in nodes], dtype=float)
+
+# PageRank
+pr_dict = nx.pagerank(G, alpha=0.85)
+pr = np.array([pr_dict[n] for n in nodes], dtype=float)
 
 def minmax(x):
     x = np.asarray(x, dtype=float)
     return (x - x.min()) / (x.max() - x.min() + 1e-8)
 
-eng_norm = minmax(engagement)
-interaction_norm = minmax(interaction_rate)
+deg_norm = minmax(deg)
+cc_norm = minmax(cc)
+avg_nd_norm = minmax(avg_nd)
+pr_norm = minmax(pr)
+
+print("Sample structural features for first 5 nodes:")
+for i in range(5):
+    print(
+        nodes[i],
+        "deg=", deg[i],
+        "deg_norm=", round(deg_norm[i], 3),
+        "cc_norm=", round(cc_norm[i], 3),
+        "avg_nd_norm=", round(avg_nd_norm[i], 3),
+        "pr_norm=", round(pr_norm[i], 3),
+    )
 
-# Trust-like base: higher interaction => more trusted
-trust_base = interaction_norm.copy()
+# -------------------------------------------------------
+# 3. Paper-style behavioural features (synthetic but graph-driven)
+# -------------------------------------------------------
 
-# Suspicious: high impressions but low engagement
-imp_norm = minmax(df["impressions"])
-suspicious_score = imp_norm * (1.0 - trust_base)
-susp_norm = minmax(suspicious_score)
+rng = np.random.default_rng(42)
 
-# Activity regularity: posts around midday more "regular"
-if "Post Hour" in df.columns:
-    post_hour = df["Post Hour"]
-else:
-    post_hour = pd.Series([12] * len(df))  # default midday if missing
+# Engagement: central users are more "engaged"
+engagement = 50 * (0.6 * deg_norm + 0.4 * avg_nd_norm) + rng.normal(0, 3, size=N)
+engagement = np.clip(engagement, 0, None)
+eng_norm = minmax(engagement)
 
-activity_reg = 1.0 - (np.abs(post_hour - 12) / 12.0).clip(0, 1)
-act_norm = minmax(activity_reg)
+# Trust base: users with higher PageRank and clustering are more trusted
+trust_base = (pr_norm + cc_norm) / 2.0
 
-# -------------------------------------------------------
-# 3. Synthetic FRR & MFR
-# -------------------------------------------------------
+# Suspicious: high degree but low clustering and low PageRank
+suspicious_raw = deg_norm * (1.0 - cc_norm) * (1.0 - pr_norm)
+suspicious_raw += 0.1 * rng.random(N)
+susp_norm = minmax(suspicious_raw)
 
-rng = np.random.default_rng(42)
+# Activity regularity: more regular if clustering is high (stable community)
+activity_reg = cc_norm + rng.normal(0, 0.05, size=N)
+activity_reg = np.clip(activity_reg, 0.0, 1.0)
+act_norm = minmax(activity_reg)
 
-# Friend requests sent (more for engaged posts)
-base_sent = rng.poisson(lam=3 + 20 * eng_norm)
-sent_requests = np.maximum(base_sent, 1)
+# Friend requests sent: more for high degree, but bounded
+sent_requests = rng.poisson(lam=2 + 15 * deg_norm)
+sent_requests = np.maximum(sent_requests, 1)
 
-# Acceptance probability depends on trust_base (0.2 to 0.9)
-accepted_prob = 0.2 + 0.7 * trust_base
+# Acceptance probability: higher for trusted, lower for suspicious
+accepted_prob = 0.1 + 0.7 * ((trust_base + (1.0 - susp_norm)) / 2.0)
 accepted_prob = np.clip(accepted_prob, 0.0, 1.0)
 accepted_requests = rng.binomial(sent_requests, accepted_prob)
 friend_request_ratio = accepted_requests / (sent_requests + 1e-8)
 frr_norm = minmax(friend_request_ratio)
 
-# Synthetic total friends
-total_friends = rng.integers(low=50, high=2000, size=len(df))
-
-# Mutual friends probability depends on trust_base
-mutual_prob = 0.1 + 0.6 * trust_base
-mutual_prob = np.clip(mutual_prob, 0.0, 1.0)
-mutual_friends = rng.binomial(total_friends, mutual_prob)
-mutual_friends_ratio = mutual_friends / (total_friends + 1e-8)
+# Mutual friends ratio (approx): we use clustering coefficient as a proxy
+# because high clustering means many mutual connections among friends.
+mutual_friends_ratio = cc_norm.copy()
 mfr_norm = minmax(mutual_friends_ratio)
 
-friends_norm = minmax(total_friends)
+friends_norm = minmax(deg)  # total friends ≈ degree
 
 # -------------------------------------------------------
-# 4. Build S, T, B scores
+# 4. Build S, T, B scores (in spirit of your paper)
 # -------------------------------------------------------
 
 # S: social / structural (FRR, MFR, friends)
@@ -106,7 +141,7 @@ T_score = (trust_base + frr_norm + (1.0 - susp_norm)) / 3.0
 B_score = (eng_norm + act_norm + susp_norm) / 3.0
 
 # -------------------------------------------------------
-# 5. Fused features with variance-based weights
+# 5. Fuse S, T, B with variance-based weights
 # -------------------------------------------------------
 
 varS = np.var(S_score)
@@ -121,8 +156,12 @@ F = np.vstack([
     wB * B_score
 ]).T  # shape (N, 3)
 
+print("Fusion weights:", wS, wT, wB)
+print("F shape:", F.shape)
+
 # -------------------------------------------------------
-# 6. Unsupervised clustering -> pseudo labels
+# 6. KMeans clustering -> pseudo labels
+#    (0 = Trusted, 1 = Under Observation, 2 = Intruder)
 # -------------------------------------------------------
 
 kmeans = KMeans(n_clusters=3, random_state=42, n_init=10)
@@ -134,9 +173,9 @@ for c in range(3):
 cluster_means_sorted = sorted(cluster_means, key=lambda x: x[1])
 
 label_map = {
-    cluster_means_sorted[0][0]: 2,  # lowest trust => Intruder
-    cluster_means_sorted[1][0]: 1,  # mid => Under Observation
-    cluster_means_sorted[2][0]: 0   # highest => Trusted
+    cluster_means_sorted[0][0]: 2,  # lowest trust → Intruder
+    cluster_means_sorted[1][0]: 1,  # medium → Under Observation
+    cluster_means_sorted[2][0]: 0   # highest → Trusted
 }
 
 cluster_labels = np.array([label_map[c] for c in cluster_raw], dtype=int)
@@ -151,22 +190,22 @@ status_counts = np.bincount(cluster_labels, minlength=3)
 
 def make_status_bar_plot():
     fig, ax = plt.subplots()
-    labels = ["Trusted", "Under Observation", "Intruder"]
-    ax.bar(labels, status_counts)
-    ax.set_ylabel("Number of posts")
-    ax.set_title("Global distribution of statuses (on dataset)")
+    labels_txt = ["Trusted", "Under Observation", "Intruder"]
+    ax.bar(labels_txt, status_counts)
+    ax.set_ylabel("Number of users")
+    ax.set_title("Global distribution of user statuses (SNAP graph)")
     fig.tight_layout()
     return fig
 
 # -------------------------------------------------------
-# 7. Train MLP on fused features
+# 7. Train small MLP on fused features -> status
 # -------------------------------------------------------
 
 X = torch.tensor(F, dtype=torch.float32)
 y = torch.tensor(cluster_labels, dtype=torch.long)
 
 dataset = TensorDataset(X, y)
-loader = DataLoader(dataset, batch_size=64, shuffle=True)
+loader = DataLoader(dataset, batch_size=128, shuffle=True)
 
 class MLPClassifier(nn.Module):
     def __init__(self, in_dim, hidden_dim=32, num_classes=3):
@@ -195,6 +234,8 @@ for epoch in range(20):
         loss.backward()
         optimizer.step()
         total_loss += loss.item() * xb.size(0)
+    # optional print, can be commented on HF to reduce logs
+    print(f"Epoch {epoch+1:02d} - loss = {total_loss / len(dataset):.4f}")
 
 model.eval()
 with torch.no_grad():
@@ -216,7 +257,7 @@ eng_min = engagement.min()
 eng_max = engagement.max()
 
 # -------------------------------------------------------
-# 8. Build S, T, B from UI inputs
+# 8. Map UI sliders -> S/T/B (paper-style logic)
 # -------------------------------------------------------
 
 def build_scores_from_user_input(
@@ -226,6 +267,7 @@ def build_scores_from_user_input(
     frr_input,
     mfr_input
 ):
+    # Normalize engagement using dataset range
     eng_norm_ui = (engagement_input - eng_min) / (eng_max - eng_min + 1e-8)
     eng_norm_ui = float(np.clip(eng_norm_ui, 0.0, 1.0))
 
@@ -234,10 +276,13 @@ def build_scores_from_user_input(
     frr_norm_ui = float(np.clip(frr_input, 0.0, 1.0))
     mfr_norm_ui = float(np.clip(mfr_input, 0.0, 1.0))
 
-    friends_norm_ui = 0.5  # fixed average friends
+    # Assume average number of friends ~ 0.5 normalized
+    friends_norm_ui = 0.5
 
+    # Trust estimate from engagement & suspiciousness
     trust_norm_ui = (eng_norm_ui + (1.0 - susp_norm_ui)) / 2.0
 
+    # Construct S / T / B
     S_ui = (frr_norm_ui + mfr_norm_ui + friends_norm_ui) / 3.0
     T_ui = (trust_norm_ui + frr_norm_ui + (1.0 - susp_norm_ui)) / 3.0
     B_ui = (eng_norm_ui + act_norm_ui + susp_norm_ui) / 3.0
@@ -245,7 +290,7 @@ def build_scores_from_user_input(
     return S_ui, T_ui, B_ui, eng_norm_ui, susp_norm_ui, act_norm_ui
 
 # -------------------------------------------------------
-# 9. Timeline helpers (T1–T5)
+# 9. Timeline (T1–T5) helpers
 # -------------------------------------------------------
 
 def make_timeline_plot(timeline_state):
@@ -257,9 +302,9 @@ def make_timeline_plot(timeline_state):
         return fig
 
     steps = [i + 1 for i in range(len(timeline_state))]
-    trusted = [e["probs"][0] for e in timeline_state]
-    obs = [e["probs"][1] for e in timeline_state]
-    intr = [e["probs"][2] for e in timeline_state]
+    trusted = [entry["probs"][0] for entry in timeline_state]
+    obs = [entry["probs"][1] for entry in timeline_state]
+    intr = [entry["probs"][2] for entry in timeline_state]
 
     ax.plot(steps, trusted, marker="o", label="Trusted")
     ax.plot(steps, obs, marker="o", label="Under Observation")
@@ -296,8 +341,9 @@ def simulate_week(
     pred, probs = predict_from_fused(S_ui, T_ui, B_ui)
     status = label_names[pred]
 
+    # Keep only last 5 time steps (T1–T5)
     if len(timeline_state) >= 5:
-        timeline_state = timeline_state[1:]  # keep only last 4
+        timeline_state = timeline_state[1:]
     timeline_state.append({
         "status": status,
         "probs": probs.tolist(),
@@ -308,9 +354,9 @@ def simulate_week(
 
     step_num = len(timeline_state)
 
-    # Current step summary
+    # Current week summary
     lines = []
-    lines.append(f"### Current Week: T{step_num}")
+    lines.append(f"### Current Time Step: T{step_num}")
     lines.append(f"**Predicted Status:** **{status}**")
     lines.append("")
     lines.append("**Probabilities:**")
@@ -354,7 +400,7 @@ def reset_timeline():
     )
 
 # -------------------------------------------------------
-# 10. Example table: real Trusted / Intruder-like samples
+# 10. Example table: real Trusted vs Intruder-like nodes
 # -------------------------------------------------------
 
 def build_example_table(n_per_class=5):
@@ -365,11 +411,10 @@ def build_example_table(n_per_class=5):
             continue
         sel = rng.choice(idxs, size=min(n_per_class, len(idxs)), replace=False)
         tmp = pd.DataFrame({
+            "NodeID": [nodes[i] for i in sel],
             "Status": [label_names[lbl]] * len(sel),
-            "Comments": df["comments"].values[sel],
-            "Likes": df["likes"].values[sel],
-            "Shares": df["shares"].values[sel],
-            "Engagement": engagement.values[sel],
+            "Degree": deg[sel],
+            "Clustering": cc[sel],
             "S_score": S_score[sel],
             "T_score": T_score[sel],
             "B_score": B_score[sel]
@@ -379,7 +424,7 @@ def build_example_table(n_per_class=5):
         return pd.concat(rows, ignore_index=True)
     else:
         return pd.DataFrame(columns=[
-            "Status", "Comments", "Likes", "Shares", "Engagement",
+            "NodeID", "Status", "Degree", "Clustering",
             "S_score", "T_score", "B_score"
         ])
 
@@ -388,7 +433,6 @@ examples_df = build_example_table()
 def refresh_examples():
     return build_example_table()
 
-# Precompute global status plot
 global_status_fig = make_status_bar_plot()
 
 # -------------------------------------------------------
@@ -396,26 +440,28 @@ global_status_fig = make_status_bar_plot()
 # -------------------------------------------------------
 
 with gr.Blocks() as demo:
-    gr.Markdown("# Trust-Based Intrusion Detection Demo (Facebook Cosmetic Brand Metrics)")
+    gr.Markdown("# Trust-Based Intrusion Detection on SNAP Facebook Graph")
     gr.Markdown(
-        "This app is trained on the **Facebook Metrics of a Cosmetic Brand** dataset.\n\n"
-        "- Real post metrics (comments, likes, shares, impressions, engaged users) are used to derive\n"
-        "  engagement, suspiciousness, and trust-like scores.\n"
-        "- Two social features – **Friend Request Ratio (FRR)** and **Mutual Friends Ratio (MFR)** –\n"
-        "  are generated synthetically but consistently with behaviour.\n\n"
-        "Use the sliders to change user behaviour. Each click on **Next week (T+1)** simulates\n"
-        "the same user at a new time step T1..T5, so you can see how their status changes over time."
+        "This demo uses the **SNAP Facebook combined graph** as a real online social network.\n\n"
+        "- Structural features (degree, clustering, PageRank, neighbour degree) come from the real graph.\n"
+        "- Behavioural features (engagement, suspiciousness, activity regularity, friend-request ratio, "
+        "mutual-friends ratio) are generated **synthetically but guided by the graph structure**, following the "
+        "spirit of your paper.\n\n"
+        "We fuse these into **S (Social)**, **T (Trust)** and **B (Behaviour)** scores, cluster users into "
+        "**Trusted / Under Observation / Intruder**, and train a small neural network to replicate this.\n\n"
+        "**Use the sliders** to simulate how a user changes behaviour over time. Each click on "
+        "**Next week (T+1)** advances the time step T1..T5 and updates the model's judgement."
     )
 
     with gr.Row():
         with gr.Column():
-            gr.Markdown("### Behaviour Inputs")
+            gr.Markdown("### Behaviour Inputs (for one user)")
             engagement_slider = gr.Slider(
                 minimum=float(eng_min),
                 maximum=float(eng_max),
                 value=float((eng_min + eng_max) / 2.0),
-                step=10.0,
-                label="Engagement level (comments + likes + shares)"
+                step=1.0,
+                label="Engagement level (synthetic, based on graph centrality)"
             )
             suspicious_slider = gr.Slider(
                 minimum=0.0,
@@ -443,7 +489,7 @@ with gr.Blocks() as demo:
                 maximum=1.0,
                 value=0.6,
                 step=0.01,
-                label="Mutual Friends Ratio"
+                label="Mutual Friends Ratio (proxy)"
             )
 
             next_button = gr.Button("Next week (T+1)")
@@ -451,7 +497,7 @@ with gr.Blocks() as demo:
 
         with gr.Column():
             current_box = gr.Markdown(
-                "Current week status will appear here after you click **Next week (T+1)**."
+                "Current time-step status will appear here after you click **Next week (T+1)**."
             )
             timeline_box = gr.Markdown(
                 "## Timeline (T1–T5)\n(No entries yet)"
@@ -461,13 +507,13 @@ with gr.Blocks() as demo:
                 label="Timeline probabilities (T1–T5)"
             )
 
-    gr.Markdown("### Global Status Distribution on Real Dataset")
+    gr.Markdown("### Global Status Distribution on the SNAP Graph")
     status_plot = gr.Plot(value=global_status_fig)
 
-    gr.Markdown("### Example Posts (Real Trusted vs Intruder-like)")
+    gr.Markdown("### Example Users (Real graph nodes: Trusted vs Intruder-like)")
     examples_table = gr.Dataframe(
         value=examples_df,
-        label="Sample posts from dataset",
+        label="Sample nodes from SNAP Facebook",
         interactive=False
     )
     refresh_button = gr.Button("Refresh examples")