visualization/ROC_curves.py

from sklearn.metrics import roc_curve
from sklearn.metrics import auc
from matplotlib import pyplot as plt
from scipy import interp

# import cleverhans
import sys
import tensorflow as tf

# from cleverhans.tf2.attacks.fast_gradient_method import fast_gradient_method
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.callbacks import Callback, LearningRateScheduler
from sklearn.model_selection import train_test_split
import pandas as pd
from sklearn.model_selection import KFold
import gzip
import pickle
import numpy as np
import warnings

warnings.filterwarnings("ignore")
import tensorflow

print("\nTensorflow Version: " + tf.__version__)
from wresnet import WideResidualNetwork
from parsevalnet import ParsevalNetwork
import hickle as hkl
import os

plt.rcParams.update({"font.size": 14})
## add your path
prefix = ""


def ROC_result(model):

    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    y_score = model.predict(X_test)
    # # Compute micro-average ROC curve and ROC area
    fpr["micro"], tpr["micro"], _ = roc_curve(y_test.ravel(), y_score.ravel())
    return fpr, tpr


def plot_roc(
    fpr_l, tpr_l, roc_auc_l, std_tpr_list, std_roc_auc_list, epsilon, fig_name
):

    plt.figure(figsize=(10, 8))
    lw = 2

    plt.fill_between(
        fpr_l,
        tpr_l[0] - std_tpr_list[0],
        tpr_l[0] + std_tpr_list[0],
        alpha=0.1,
        color="purple",
    )
    plt.plot(
        fpr_l,
        tpr_l[0],
        color="purple",
        lw=lw,
        label="ResNet (area = %0.4f)" % roc_auc_l[0],
    )

    plt.fill_between(
        fpr_l,
        tpr_l[1] - std_tpr_list[1],
        tpr_l[1] + std_tpr_list[1],
        alpha=0.1,
        color="darkgreen",
    )
    plt.plot(
        fpr_l,
        tpr_l[1],
        color="darkgreen",
        lw=lw,
        label="ResNet-0.25 AEs(area = %0.4f)" % roc_auc_l[1],
    )

    plt.fill_between(
        fpr_l,
        tpr_l[2] - std_tpr_list[2],
        tpr_l[2] + std_tpr_list[2],
        alpha=0.1,
        color="darkblue",
    )
    plt.plot(
        fpr_l,
        tpr_l[2],
        color="darkblue",
        lw=lw,
        label="ResNet-0.50 AEs(area = %0.4f)" % roc_auc_l[2],
    )

    plt.fill_between(
        fpr_l,
        tpr_l[3] - std_tpr_list[3],
        tpr_l[3] + std_tpr_list[3],
        alpha=0.1,
        color="pink",
    )
    plt.plot(
        fpr_l,
        tpr_l[3],
        color="pink",
        lw=lw,
        label="ResNet-0.75 AEs(area = %0.4f)" % roc_auc_l[3],
    )

    plt.fill_between(
        fpr_l,
        tpr_l[4] - std_tpr_list[4],
        tpr_l[4] + std_tpr_list[4],
        alpha=0.1,
        color="darkorange",
    )
    plt.plot(
        fpr_l,
        tpr_l[4],
        color="darkorange",
        lw=lw,
        label="ResNet-1.0 AEs (area = %0.4f)" % roc_auc_l[4],
    )

    plt.fill_between(
        fpr_l,
        tpr_l[5] - std_tpr_list[5],
        tpr_l[5] + std_tpr_list[5],
        alpha=0.1,
        color="darkmagenta",
    )
    plt.plot(
        fpr_l,
        tpr_l[5],
        color="darkmagenta",
        lw=lw,
        label="Parseval (area = %0.4f)" % roc_auc_l[5],
    )

    plt.plot(np.arange(0, 1, 0.001), np.arange(0, 1, 0.001), linestyle="--")

    plt.xlabel("False Positive Rate")
    plt.ylabel("True Positive Rate")
    plt.title("ROC For Epsilon = {}".format(epsilon))
    plt.legend(loc="lower right")

    plt.xscale("log")

    plt.savefig(fig_name)


BS = 64
init = (32, 32, 1)
sgd = SGD(lr=0.1, momentum=0.9)

data = hkl.load("data.hkl")
X_train, X_test, Y_train, y_test = (
    data["xtrain"],
    data["xtest"],
    data["ytrain"],
    data["ytest"],
)

Experiment = ["AEModels", "RandomNoisemodels"]
epsilons_list = [0.03, 0.01, 0.005, 0.003, 0.001]
percent_list = [0, 0.25, 0.5, 0.75, 1.0]

for exp in Experiment:

    for epsilon in epsilons_list:
        for percent in percent_list:
            tprs = []
            aucs = []
            mean_fpr = np.arange(0, 1, 0.001)
            mean_fpr_list = []
            mean_tpr_list = []
            mean_roc_auc_list = []
            std_fpr_list = []
            std_tpr_list = []
            std_roc_auc_list = []
            micro_roc_auc = []
            fpr_list, tpr_list, roc_auc_list = [], [], []
            for i in range(10):
                resnet = WideResidualNetwork(
                    init, 0.0001, 0.9, nb_classes=4, N=2, k=1, dropout=0.0
                )
                model = resnet.create_wide_residual_network()
                model.compile(
                    loss="categorical_crossentropy", optimizer=sgd, metrics=["acc"]
                )
                if percent != 0:
                    model_path = (
                        prefix
                        + ex
                        + "/ResNet_"
                        + str(epsilon)
                        + "_"
                        + str(percent)
                        + "_"
                        + str(i)
                        + ".h5"
                    )
                    print(model_path)
                    model.load_weights(model_path)
                else:
                    model_path = prefix + "ResNet/ResNet_" + str(i) + ".h5"
                    print(model_path)
                    model.load_weights(model_path)
                    fpr, tpr = ROC_result(model)
                    tprs.append(interp(mean_fpr, fpr["micro"], tpr["micro"]))
                    roc_auc = auc(fpr["micro"], tpr["micro"])
                    aucs.append(roc_auc)
                    tpr_list.append(tprs)
                    roc_auc_list.append(aucs)
                    init = (32, 32, 1)
                    parseval_micro_fpr, parseval_micro_tpr, parseval_micro_roc_auc = (
                        [],
                        [],
                        [],
                    )
for i in range(10):
    parseval = ParsevalNetwork(init, 0.0001, 0.9, nb_classes=4, N=2, k=1, dropout=0.0)

    model = parseval.create_wide_residual_network()
    model.compile(loss="categorical_crossentropy", optimizer=sgd, metrics=["acc"])
    model_name = prefix + "ResNet/Parseval_" + str(i) + ".h5"
    model.load_weights(model_name)
    fpr, tpr = ROC_result(model)
    parseval_micro_tpr.append(interp(mean_fpr, fpr["micro"], tpr["micro"]))
    roc_auc = auc(fpr["micro"], tpr["micro"])
    parseval_micro_roc_auc.append(roc_auc)
    parseval_micro_roc_auc.append(roc_auc)
    tpr_list.append(parseval_micro_tpr)
    roc_auc_list.append(parseval_micro_roc_auc)

    for i in range(6):
        mean_tpr_list.append(np.mean(tpr_list[i], axis=0))
        mean_roc_auc_list.append(auc(mean_fpr, mean_tpr_list[i]))
        std_tpr_list.append(np.std(tpr_list[i], axis=0))
        std_roc_auc_list.append(auc(mean_fpr, std_tpr_list[i]))

    fig_name = prefix + exp + "/ROC/Model_Epsilon" + str(epsilon) + ".png"
    plot_roc(
        mean_fpr,
        mean_tpr_list,
        mean_roc_auc_list,
        std_tpr_list,
        std_roc_auc_list,
        epsilon,
        fig_name,
    )