import os import time import torch import numpy as np import torch.nn.functional as F import matplotlib.pyplot as plt from torch.utils.data import DataLoader from torchvision import datasets, transforms from transformers import ViTForImageClassification, ViTConfig from sklearn.metrics import ( accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix, roc_curve ) from openpyxl import Workbook def main(): # ---------------------------------- # PERFORMANCE TUNING # ---------------------------------- torch.backends.cudnn.benchmark = True torch.set_num_threads(8) # ---------------------------------- # Device (SAFE) # ---------------------------------- device = torch.device("cuda" if torch.cuda.is_available() else "cpu") use_amp = device.type == "cuda" if use_amp: print(f"๐Ÿš€ Using device: cuda ({torch.cuda.get_device_name(0)})") else: print("๐Ÿš€ Using device: cpu") # ---------------------------------- # Paths # ---------------------------------- base_data_dir = "data" # ๐Ÿ”ด UPDATED MODEL PATH (NEW MODEL) model_path = "model/vit_face_final_best.pth" datasets_config = { "FF++": f"{base_data_dir}/ff++/test", "Celeb-DF": f"{base_data_dir}/celeb-df/test", "DFDC": f"{base_data_dir}/dfdc/test", } os.makedirs("outputs", exist_ok=True) # ---------------------------------- # Transforms # ---------------------------------- test_tfms = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) ]) # ---------------------------------- # Model # ---------------------------------- config = ViTConfig.from_pretrained( "google/vit-base-patch16-224", num_labels=2 ) model = ViTForImageClassification.from_pretrained( "google/vit-base-patch16-224", config=config, ignore_mismatched_sizes=True ) model.load_state_dict(torch.load(model_path, map_location=device)) model.to(device) model.eval() print("โœ… Model loaded successfully") # ---------------------------------- # Evaluation # ---------------------------------- all_results = {} print("\n๐Ÿงช Running Cross-Dataset Evaluation...\n") for ds_name, test_dir in datasets_config.items(): if not os.path.exists(test_dir): print(f"โš ๏ธ {ds_name}: path not found โ€” skipping") continue test_ds = datasets.ImageFolder(test_dir, transform=test_tfms) test_dl = DataLoader( test_ds, batch_size=32, shuffle=False, num_workers=2, # SAFE (inside main) pin_memory=True ) class_to_idx = test_ds.class_to_idx real_idx = class_to_idx["real"] fake_idx = class_to_idx["fake"] print(f"๐Ÿ“‚ {ds_name}: {len(test_ds)} samples | {class_to_idx}") y_true, y_pred, y_probs = [], [], [] total_images = 0 total_time = 0.0 with torch.no_grad(): for imgs, labels in test_dl: imgs = imgs.to(device, non_blocking=True) labels = labels.to(device, non_blocking=True) start = time.time() with torch.amp.autocast(device_type="cuda", enabled=use_amp): logits = model(imgs).logits end = time.time() probs = F.softmax(logits, dim=1)[:, fake_idx] preds = torch.argmax(logits, dim=1) total_time += (end - start) total_images += imgs.size(0) y_true.extend(labels.cpu().numpy()) y_pred.extend(preds.cpu().numpy()) y_probs.extend(probs.cpu().numpy()) # ---------------- Metrics ---------------- acc = accuracy_score(y_true, y_pred) prec = precision_score(y_true, y_pred, zero_division=0) rec = recall_score(y_true, y_pred, zero_division=0) f1 = f1_score(y_true, y_pred, zero_division=0) try: auc = roc_auc_score( (np.array(y_true) == fake_idx).astype(int), y_probs ) except ValueError: auc = float("nan") avg_time_ms = (total_time / total_images) * 1000 fps = total_images / total_time all_results[ds_name] = { "acc": acc, "prec": prec, "rec": rec, "f1": f1, "auc": auc, "time": avg_time_ms, "fps": fps, } print( f"๐ŸŽฏ {ds_name} | Acc: {acc:.4f} | F1: {f1:.4f} | " f"AUC: {auc:.4f} | {avg_time_ms:.2f} ms/img | {fps:.2f} FPS" ) # ---------------- Confusion Matrix ---------------- cm = confusion_matrix(y_true, y_pred, labels=[real_idx, fake_idx]) plt.figure(figsize=(5, 4)) plt.imshow(cm) plt.title(f"{ds_name} - Confusion Matrix") plt.xlabel("Predicted") plt.ylabel("True") plt.xticks([0, 1], ["Real", "Fake"]) plt.yticks([0, 1], ["Real", "Fake"]) for i in range(2): for j in range(2): plt.text(j, i, cm[i, j], ha="center", va="center") plt.tight_layout() plt.savefig(f"outputs/cm_{ds_name}.png") plt.close() # ---------------- ROC Curve ---------------- fpr, tpr, _ = roc_curve(y_true, y_probs, pos_label=fake_idx) plt.figure(figsize=(5, 4)) plt.plot(fpr, tpr, label=f"AUC = {auc:.4f}") plt.plot([0, 1], [0, 1], linestyle="--") plt.xlabel("False Positive Rate") plt.ylabel("True Positive Rate") plt.title(f"{ds_name} - ROC Curve") plt.legend(loc="lower right") plt.tight_layout() plt.savefig(f"outputs/roc_{ds_name}.png") plt.close() # ---------------------------------- # EXPORT (EXCEL + TEXT) # ---------------------------------- avg_metrics = { "acc": np.mean([m["acc"] for m in all_results.values()]), "prec": np.mean([m["prec"] for m in all_results.values()]), "rec": np.mean([m["rec"] for m in all_results.values()]), "f1": np.mean([m["f1"] for m in all_results.values()]), "auc": np.nanmean([m["auc"] for m in all_results.values()]), "time": np.mean([m["time"] for m in all_results.values()]), "fps": np.mean([m["fps"] for m in all_results.values()]), } all_results["AVERAGE"] = avg_metrics wb = Workbook() ws = wb.active ws.title = "Evaluation Results" ws.append(["Dataset", "Accuracy", "Precision", "Recall", "F1", "AUC", "ms/img", "FPS"]) for ds, m in all_results.items(): ws.append([ ds, round(m["acc"], 4), round(m["prec"], 4), round(m["rec"], 4), round(m["f1"], 4), round(m["auc"], 4), round(m["time"], 2), round(m["fps"], 2), ]) wb.save("outputs/evaluation_results.xlsx") with open("outputs/summary.txt", "w") as f: for ds, m in all_results.items(): f.write(f"Dataset: {ds}\n") for k, v in m.items(): f.write(f" {k.upper():<6}: {v}\n") f.write("-" * 45 + "\n") print("โœ… Evaluation complete (new model evaluated successfully)") # ---------------------------------- # COMBINED ROC CURVE (IEEE-FRIENDLY) # ---------------------------------- plt.figure(figsize=(5, 4)) for ds_name in datasets_config.keys(): if ds_name not in all_results: continue # Reload dataset to recompute ROC cleanly test_dir = datasets_config[ds_name] test_ds = datasets.ImageFolder(test_dir, transform=test_tfms) class_to_idx = test_ds.class_to_idx fake_idx = class_to_idx["fake"] test_dl = DataLoader( test_ds, batch_size=32, shuffle=False, num_workers=2, pin_memory=True ) y_true, y_probs = [], [] with torch.no_grad(): for imgs, labels in test_dl: imgs = imgs.to(device, non_blocking=True) labels = labels.to(device, non_blocking=True) with torch.amp.autocast(device_type="cuda", enabled=use_amp): logits = model(imgs).logits probs = F.softmax(logits, dim=1)[:, fake_idx] y_true.extend((labels == fake_idx).cpu().numpy()) y_probs.extend(probs.cpu().numpy()) fpr, tpr, _ = roc_curve(y_true, y_probs) auc_val = roc_auc_score(y_true, y_probs) # Line styles for IEEE (print-safe) if ds_name == "FF++": style = "-" elif ds_name == "Celeb-DF": style = "--" else: # DFDC style = "-." plt.plot( fpr, tpr, linestyle=style, linewidth=2, label=f"{ds_name} (AUC = {auc_val:.4f})" ) # Random baseline plt.plot([0, 1], [0, 1], linestyle=":", linewidth=1) plt.xlabel("False Positive Rate") plt.ylabel("True Positive Rate") plt.legend(loc="lower right") plt.tight_layout() # Save IEEE-ready figure plt.savefig("outputs/roc_combined.png", dpi=300, bbox_inches="tight") plt.savefig("outputs/roc_combined.pdf", dpi=300, bbox_inches="tight") plt.close() if __name__ == "__main__": main()