ai-backend/eval_model.py

"""
eval_model.py — NLP Model Evaluation for Dark Patterns Detector
================================================================
Drop this file into the ai-backend/ folder alongside dataset.csv and run:

    pip install scikit-learn pandas matplotlib seaborn
    python eval_model.py

Outputs:
  • Classification report (precision / recall / F1 per class)
  • Overall accuracy
  • Confusion matrix saved as confusion_matrix.png
  • Model comparison table (baseline vs improved TF-IDF settings)
"""

import pandas as pd
import matplotlib
matplotlib.use("Agg")          # headless — no display needed
import matplotlib.pyplot as plt
import seaborn as sns
import os, sys

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import (
    classification_report, confusion_matrix, accuracy_score
)

# ── Load dataset ──────────────────────────────────────────────────────
script_dir = os.path.dirname(os.path.abspath(__file__))
dataset_path = os.path.join(script_dir, "dataset.csv")

if not os.path.exists(dataset_path):
    print("❌  dataset.csv not found. Make sure this script is in ai-backend/")
    sys.exit(1)

df = pd.read_csv(dataset_path).dropna(subset=["text", "Pattern Category"])
X = df["text"]
y = df["Pattern Category"]

print(f"✅  Loaded {len(df)} samples across {y.nunique()} classes\n")
print("Class distribution:")
print(y.value_counts().to_string())
print()

# ── Train / test split ────────────────────────────────────────────────
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.20, random_state=42, stratify=y
)

# ── Model 1: Baseline (as in production) ─────────────────────────────
model_baseline = make_pipeline(
    TfidfVectorizer(ngram_range=(1, 2)),
    LogisticRegression(C=10.0, class_weight="balanced", max_iter=1000)
)
model_baseline.fit(X_train, y_train)
y_pred_baseline = model_baseline.predict(X_test)

# ── Model 2: Improved (sublinear TF scaling + min_df pruning) ─────────
model_improved = make_pipeline(
    TfidfVectorizer(
        ngram_range=(1, 3),
        sublinear_tf=True,
        min_df=2,
        max_features=50_000,
    ),
    LogisticRegression(C=5.0, class_weight="balanced", max_iter=1000, solver="saga")
)
model_improved.fit(X_train, y_train)
y_pred_improved = model_improved.predict(X_test)

# ── Print reports ─────────────────────────────────────────────────────
print("=" * 65)
print("BASELINE MODEL  —  TF-IDF(1,2) + LogReg(C=10)")
print("=" * 65)
print(f"Accuracy: {accuracy_score(y_test, y_pred_baseline):.4f}\n")
print(classification_report(y_test, y_pred_baseline))

print("=" * 65)
print("IMPROVED MODEL  —  TF-IDF(1,3, sublinear) + LogReg(C=5, saga)")
print("=" * 65)
print(f"Accuracy: {accuracy_score(y_test, y_pred_improved):.4f}\n")
print(classification_report(y_test, y_pred_improved))

# ── 5-fold cross-validation ───────────────────────────────────────────
cv_baseline = cross_val_score(model_baseline, X, y, cv=5, scoring="accuracy")
cv_improved  = cross_val_score(model_improved,  X, y, cv=5, scoring="accuracy")

print("5-Fold Cross-Validation Accuracy:")
print(f"  Baseline : {cv_baseline.mean():.4f} ± {cv_baseline.std():.4f}")
print(f"  Improved : {cv_improved.mean():.4f}  ± {cv_improved.std():.4f}")
print()

# ── Confusion matrix plot ─────────────────────────────────────────────
labels = sorted(y.unique())
cm = confusion_matrix(y_test, y_pred_improved, labels=labels)

fig, ax = plt.subplots(figsize=(10, 8))
sns.heatmap(
    cm,
    annot=True, fmt="d", cmap="Blues",
    xticklabels=labels, yticklabels=labels,
    linewidths=0.5, linecolor="white",
    ax=ax
)
ax.set_title("Confusion Matrix — Improved Model", fontsize=14, fontweight="bold", pad=14)
ax.set_ylabel("True Label", fontsize=12)
ax.set_xlabel("Predicted Label", fontsize=12)
plt.xticks(rotation=30, ha="right", fontsize=9)
plt.yticks(rotation=0, fontsize=9)
plt.tight_layout()

out_path = os.path.join(script_dir, "confusion_matrix.png")
plt.savefig(out_path, dpi=150)
print(f"📊  Confusion matrix saved → {out_path}")

# ── Top features per class ────────────────────────────────────────────
print("\n── Top 8 TF-IDF features per class (Improved Model) ──")
tfidf   = model_improved.named_steps["tfidfvectorizer"]
logreg  = model_improved.named_steps["logisticregression"]
feat_names = tfidf.get_feature_names_out()

for i, cls in enumerate(logreg.classes_):
    top_idx = logreg.coef_[i].argsort()[-8:][::-1]
    top_feats = [feat_names[j] for j in top_idx]
    print(f"  {cls:<20}: {', '.join(top_feats)}")

print("\n✅  Evaluation complete.")