src/eval/plots.py

"""Generate plots for Chapter 5.

Reads the CSVs produced by src/meta/train_eval.py and src/meta/logo_eval.py
and writes a set of figures to outputs/figures/. Each figure is a single
visual claim that can be dropped into Chapter 5 with a caption.
"""
from __future__ import annotations

import argparse
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.calibration import calibration_curve
from sklearn.metrics import roc_curve

sns.set_theme(context="paper", style="whitegrid", palette="colorblind")
plt.rcParams["figure.dpi"] = 120
plt.rcParams["savefig.dpi"] = 200
plt.rcParams["font.family"] = "DejaVu Sans"

SYSTEM_ORDER = [
    "binoculars_solo",
    "fast_detect_gpt_solo",
    "roberta_solo",
    "meta_logistic",
    "meta_xgboost",
    "meta_mlp",
]
SYSTEM_LABELS = {
    "binoculars_solo": "Binoculars",
    "fast_detect_gpt_solo": "Fast-DetectGPT",
    "roberta_solo": "RoBERTa",
    "meta_logistic": "Meta (logistic)",
    "meta_xgboost": "Meta (XGBoost)",
    "meta_mlp": "Meta (MLP)",
}


def plot_roc_curves(predictions: pd.DataFrame, outdir: Path) -> None:
    fig, ax = plt.subplots(figsize=(6, 5))
    for system in SYSTEM_ORDER:
        if system not in predictions.columns:
            continue
        fpr, tpr, _ = roc_curve(predictions["label"], predictions[system])
        ax.plot(fpr, tpr, label=SYSTEM_LABELS[system], linewidth=1.8)
    ax.plot([0, 1], [0, 1], "k--", alpha=0.4, linewidth=0.8, label="Random")
    ax.set_xlabel("False positive rate")
    ax.set_ylabel("True positive rate")
    ax.set_title("ROC curves on RAID test set (n = {:,})".format(len(predictions)))
    ax.legend(loc="lower right", frameon=True)
    ax.set_xlim(0, 1)
    ax.set_ylim(0, 1.02)
    fig.tight_layout()
    fig.savefig(outdir / "fig_roc_curves.png")
    plt.close(fig)
    print("  fig_roc_curves.png")


def plot_fpr_at_tpr95(summary: pd.DataFrame, outdir: Path) -> None:
    df = summary.copy()
    df["display"] = df["system"].map(SYSTEM_LABELS)
    df = df.sort_values("fpr_at_tpr_95", ascending=False)

    fig, ax = plt.subplots(figsize=(7, 4))
    colours = ["#d62728" if "solo" in s else "#1f77b4" for s in df["system"]]
    bars = ax.barh(df["display"], df["fpr_at_tpr_95"], color=colours, edgecolor="black", linewidth=0.5)
    ax.set_xlabel("False positive rate at TPR = 0.95")
    ax.set_title("False positive cost of high-recall detection (lower is better)")
    ax.set_xlim(0, 1.05)
    for bar, value in zip(bars, df["fpr_at_tpr_95"]):
        ax.text(value + 0.01, bar.get_y() + bar.get_height() / 2,
                f"{value:.3f}", va="center", fontsize=9)
    fig.tight_layout()
    fig.savefig(outdir / "fig_fpr_at_tpr95.png")
    plt.close(fig)
    print("  fig_fpr_at_tpr95.png")


def plot_score_distributions(predictions: pd.DataFrame, outdir: Path) -> None:
    base = ["binoculars_solo", "fast_detect_gpt_solo", "roberta_solo"]
    fig, axes = plt.subplots(1, 3, figsize=(12, 3.5), sharey=False)
    for ax, system in zip(axes, base):
        for label, sub in predictions.groupby("label"):
            sns.histplot(sub[system], bins=40, ax=ax, alpha=0.55,
                         label="Human" if label == 0 else "AI",
                         element="step", stat="density", common_norm=False)
        ax.set_title(SYSTEM_LABELS[system])
        ax.set_xlabel("Score (normalised to [0, 1])")
        ax.set_ylabel("Density")
        ax.legend(loc="upper center")
    fig.suptitle("Base detector score distributions on test set")
    fig.tight_layout()
    fig.savefig(outdir / "fig_score_distributions.png")
    plt.close(fig)
    print("  fig_score_distributions.png")


def plot_calibration(predictions: pd.DataFrame, outdir: Path) -> None:
    fig, ax = plt.subplots(figsize=(6, 5))
    for system in SYSTEM_ORDER:
        if system not in predictions.columns:
            continue
        try:
            prob_true, prob_pred = calibration_curve(
                predictions["label"], predictions[system], n_bins=15, strategy="uniform"
            )
            ax.plot(prob_pred, prob_true, marker="o", linewidth=1.5,
                    label=SYSTEM_LABELS[system])
        except Exception as exc:  # pragma: no cover
            print(f"    skip {system}: {exc}")
    ax.plot([0, 1], [0, 1], "k--", alpha=0.4, label="Perfect calibration")
    ax.set_xlabel("Predicted probability of AI")
    ax.set_ylabel("Empirical frequency of AI")
    ax.set_title("Reliability diagram (15-bin)")
    ax.legend(loc="upper left", fontsize=8)
    ax.set_xlim(0, 1)
    ax.set_ylim(0, 1)
    fig.tight_layout()
    fig.savefig(outdir / "fig_calibration.png")
    plt.close(fig)
    print("  fig_calibration.png")


def plot_logo_results(logo: pd.DataFrame, outdir: Path) -> None:
    df = logo.sort_values("auroc")
    x = np.arange(len(df))
    width = 0.4

    fig, ax1 = plt.subplots(figsize=(8, 4.5))
    ax2 = ax1.twinx()

    bars1 = ax1.bar(x - width / 2, df["auroc"], width, color="#1f77b4",
                    label="AUROC", edgecolor="black", linewidth=0.4)
    bars2 = ax2.bar(x + width / 2, df["fpr_at_tpr_95"], width, color="#d62728",
                    label="FPR@TPR=0.95", edgecolor="black", linewidth=0.4)

    ax1.set_xticks(x)
    ax1.set_xticklabels(df["held_out_generator"], rotation=35, ha="right")
    ax1.set_ylim(0.5, 1.02)
    ax2.set_ylim(0, 1.05)
    ax1.set_ylabel("AUROC", color="#1f77b4")
    ax2.set_ylabel("FPR at TPR = 0.95", color="#d62728")
    ax1.tick_params(axis="y", labelcolor="#1f77b4")
    ax2.tick_params(axis="y", labelcolor="#d62728")
    ax1.set_title("Leave-one-generator-out: meta-classifier (XGBoost) on unseen generator")
    ax1.legend(loc="upper left")
    ax2.legend(loc="upper right")
    fig.tight_layout()
    fig.savefig(outdir / "fig_logo_per_generator.png")
    plt.close(fig)
    print("  fig_logo_per_generator.png")


def plot_per_domain_heatmap(per_domain: pd.DataFrame, outdir: Path) -> None:
    pivot = per_domain.pivot(index="system", columns="domain", values="auroc")
    pivot = pivot.reindex([s for s in SYSTEM_ORDER if s in pivot.index])
    pivot.index = [SYSTEM_LABELS[s] for s in pivot.index]

    fig, ax = plt.subplots(figsize=(8, 4.5))
    sns.heatmap(pivot, annot=True, fmt=".3f", cmap="RdYlGn", vmin=0.5, vmax=1.0,
                cbar_kws={"label": "AUROC"}, ax=ax, linewidths=0.4, linecolor="white")
    ax.set_title("AUROC per (system, domain)")
    ax.set_xlabel("Domain")
    ax.set_ylabel("System")
    plt.setp(ax.get_xticklabels(), rotation=30, ha="right")
    fig.tight_layout()
    fig.savefig(outdir / "fig_per_domain_heatmap.png")
    plt.close(fig)
    print("  fig_per_domain_heatmap.png")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--outdir", type=str, default="outputs")
    args = parser.parse_args()

    outdir = Path(args.outdir)
    figdir = outdir / "figures"
    figdir.mkdir(parents=True, exist_ok=True)

    summary = pd.read_csv(outdir / "metrics_summary.csv")
    predictions = pd.read_csv(outdir / "predictions_test.csv")
    per_domain = pd.read_csv(outdir / "per_domain.csv")
    logo = pd.read_csv(outdir / "logo_results.csv")

    print("Generating figures:")
    plot_roc_curves(predictions, figdir)
    plot_fpr_at_tpr95(summary, figdir)
    plot_score_distributions(predictions, figdir)
    plot_calibration(predictions, figdir)
    plot_logo_results(logo, figdir)
    plot_per_domain_heatmap(per_domain, figdir)
    print(f"\nAll figures saved to {figdir}")


if __name__ == "__main__":
    main()