Add eval_model.py for reproducible offline evaluation

ai-backend/eval_model.py

@@ -0,0 +1,130 @@
+"""
+eval_model.py — NLP Model Evaluation for Dark Patterns Detector
+================================================================
+Drop this file into the ai-backend/ folder alongside dataset.csv and run:
+
+    pip install scikit-learn pandas matplotlib seaborn
+    python eval_model.py
+
+Outputs:
+  • Classification report (precision / recall / F1 per class)
+  • Overall accuracy
+  • Confusion matrix saved as confusion_matrix.png
+  • Model comparison table (baseline vs improved TF-IDF settings)
+"""
+
+import pandas as pd
+import matplotlib
+matplotlib.use("Agg")          # headless — no display needed
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os, sys
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import LogisticRegression
+from sklearn.pipeline import make_pipeline
+from sklearn.model_selection import train_test_split, cross_val_score
+from sklearn.metrics import (
+    classification_report, confusion_matrix, accuracy_score
+)
+
+# ── Load dataset ──────────────────────────────────────────────────────
+script_dir = os.path.dirname(os.path.abspath(__file__))
+dataset_path = os.path.join(script_dir, "dataset.csv")
+
+if not os.path.exists(dataset_path):
+    print("❌  dataset.csv not found. Make sure this script is in ai-backend/")
+    sys.exit(1)
+
+df = pd.read_csv(dataset_path).dropna(subset=["text", "Pattern Category"])
+X = df["text"]
+y = df["Pattern Category"]
+
+print(f"✅  Loaded {len(df)} samples across {y.nunique()} classes\n")
+print("Class distribution:")
+print(y.value_counts().to_string())
+print()
+
+# ── Train / test split ────────────────────────────────────────────────
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.20, random_state=42, stratify=y
+)
+
+# ── Model 1: Baseline (as in production) ─────────────────────────────
+model_baseline = make_pipeline(
+    TfidfVectorizer(ngram_range=(1, 2)),
+    LogisticRegression(C=10.0, class_weight="balanced", max_iter=1000)
+)
+model_baseline.fit(X_train, y_train)
+y_pred_baseline = model_baseline.predict(X_test)
+
+# ── Model 2: Improved (sublinear TF scaling + min_df pruning) ─────────
+model_improved = make_pipeline(
+    TfidfVectorizer(
+        ngram_range=(1, 3),
+        sublinear_tf=True,
+        min_df=2,
+        max_features=50_000,
+    ),
+    LogisticRegression(C=5.0, class_weight="balanced", max_iter=1000, solver="saga")
+)
+model_improved.fit(X_train, y_train)
+y_pred_improved = model_improved.predict(X_test)
+
+# ── Print reports ─────────────────────────────────────────────────────
+print("=" * 65)
+print("BASELINE MODEL  —  TF-IDF(1,2) + LogReg(C=10)")
+print("=" * 65)
+print(f"Accuracy: {accuracy_score(y_test, y_pred_baseline):.4f}\n")
+print(classification_report(y_test, y_pred_baseline))
+
+print("=" * 65)
+print("IMPROVED MODEL  —  TF-IDF(1,3, sublinear) + LogReg(C=5, saga)")
+print("=" * 65)
+print(f"Accuracy: {accuracy_score(y_test, y_pred_improved):.4f}\n")
+print(classification_report(y_test, y_pred_improved))
+
+# ── 5-fold cross-validation ───────────────────────────────────────────
+cv_baseline = cross_val_score(model_baseline, X, y, cv=5, scoring="accuracy")
+cv_improved  = cross_val_score(model_improved,  X, y, cv=5, scoring="accuracy")
+
+print("5-Fold Cross-Validation Accuracy:")
+print(f"  Baseline : {cv_baseline.mean():.4f} ± {cv_baseline.std():.4f}")
+print(f"  Improved : {cv_improved.mean():.4f}  ± {cv_improved.std():.4f}")
+print()
+
+# ── Confusion matrix plot ─────────────────────────────────────────────
+labels = sorted(y.unique())
+cm = confusion_matrix(y_test, y_pred_improved, labels=labels)
+
+fig, ax = plt.subplots(figsize=(10, 8))
+sns.heatmap(
+    cm,
+    annot=True, fmt="d", cmap="Blues",
+    xticklabels=labels, yticklabels=labels,
+    linewidths=0.5, linecolor="white",
+    ax=ax
+)
+ax.set_title("Confusion Matrix — Improved Model", fontsize=14, fontweight="bold", pad=14)
+ax.set_ylabel("True Label", fontsize=12)
+ax.set_xlabel("Predicted Label", fontsize=12)
+plt.xticks(rotation=30, ha="right", fontsize=9)
+plt.yticks(rotation=0, fontsize=9)
+plt.tight_layout()
+
+out_path = os.path.join(script_dir, "confusion_matrix.png")
+plt.savefig(out_path, dpi=150)
+print(f"📊  Confusion matrix saved → {out_path}")
+
+# ── Top features per class ────────────────────────────────────────────
+print("\n── Top 8 TF-IDF features per class (Improved Model) ──")
+tfidf   = model_improved.named_steps["tfidfvectorizer"]
+logreg  = model_improved.named_steps["logisticregression"]
+feat_names = tfidf.get_feature_names_out()
+
+for i, cls in enumerate(logreg.classes_):
+    top_idx = logreg.coef_[i].argsort()[-8:][::-1]
+    top_feats = [feat_names[j] for j in top_idx]
+    print(f"  {cls:<20}: {', '.join(top_feats)}")
+
+print("\n✅  Evaluation complete.")