last changes, hopefully!

.gitattributes

@@ -1,6 +1,3 @@
-<<<<<<< HEAD
-checkpoints/model_best.pt filter=lfs diff=lfs merge=lfs -text
-=======
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
@@ -36,5 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
->>>>>>> 10b34e9e01a793df83cca1499ece5c6b29f10a90
 checkpoints/model.pth filter=lfs diff=lfs merge=lfs -text
+checkpoints/efficientnet.onnx filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+checkpoints/efficientnet.onnx
+checkpoints/RawNet2.pth

checkpoints/model.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd7e7092a26ba6b2927a05150d25f03fb19e4562006835cfa585a055b419f2f2
-size 604878654
+oid sha256:3de812710093068acee6200b8d162aab074975edffa3edf2ccbe562868e4adf6
+size 117418889

inference_2.py

@@ -1,5 +1,6 @@
 import os
 import cv2
+import onnx
 import torch
 import argparse
 import numpy as np
@@ -7,6 +8,11 @@ import torch.nn as nn
 from models.TMC import ETMC
 from models import image
 
+from onnx2pytorch import ConvertModel
+
+onnx_model = onnx.load('checkpoints\\efficientnet.onnx')
+pytorch_model = ConvertModel(onnx_model)
+
 #Set random seed for reproducibility.
 torch.manual_seed(42)
 
@@ -70,28 +76,29 @@ def load_multimodal_model(args):
     '''Load multimodal model'''
     model = ETMC(args)
     ckpt = torch.load('checkpoints\\model.pth', map_location = torch.device('cpu'))
-    model.load_state_dict(ckpt,strict = True)
+    model.load_state_dict(ckpt, strict = True)
     model.eval()
     return model
 
 def load_img_modality_model(args):
     '''Loads image modality model.'''
-    rgb_encoder = image.ImageEncoder(args)
-    ckpt = torch.load('checkpoints/model.pth', map_location = torch.device('cpu'))
-    rgb_encoder.load_state_dict(ckpt, strict = False)
+    rgb_encoder = pytorch_model
+
+    ckpt = torch.load('checkpoints\\model.pth', map_location = torch.device('cpu'))
+    rgb_encoder.load_state_dict(ckpt['rgb_encoder'], strict = True)
     rgb_encoder.eval()
     return rgb_encoder
 
 def load_spec_modality_model(args):
     spec_encoder = image.RawNet(args)
     ckpt = torch.load('checkpoints/model.pth', map_location = torch.device('cpu'))
-    spec_encoder.load_state_dict(ckpt, strict = False)
+    spec_encoder.load_state_dict(ckpt['spec_encoder'], strict = True)
     spec_encoder.eval()
     return spec_encoder
 
 
 #Load models.
-parser = argparse.ArgumentParser(description="Train Models")
+parser = argparse.ArgumentParser(description="Inference models")
 get_args(parser)
 args, remaining_args = parser.parse_known_args()
 assert remaining_args == [], remaining_args
@@ -104,7 +111,7 @@ img_model = load_img_modality_model(args)
 def preprocess_img(face):
     face = face / 255
     face = cv2.resize(face, (256, 256))
-    face = face.transpose(2, 0, 1) #(W, H, C) -> (C, W, H)
+    # face = face.transpose(2, 0, 1) #(W, H, C) -> (C, W, H)
     face_pt = torch.unsqueeze(torch.Tensor(face), dim = 0) 
     return face_pt
 
@@ -137,18 +144,18 @@ def deepfakes_spec_predict(input_audio):
 
 def deepfakes_image_predict(input_image):
     face = preprocess_img(input_image)
-
+    print(f"Face shape is: {face.shape}")
     img_grads = img_model.forward(face)
     img_grads = img_grads.cpu().detach().numpy()
     img_grads_np = np.squeeze(img_grads)
 
-    if img_grads_np > 0.5:
-        preds = round(100 - (img_grads_np * 100), 3)
-        text2 = f"The image is REAL."
+    if img_grads_np[0] > 0.5:
+        preds = round(img_grads_np[0] * 100, 3)
+        text2 = f"The image is REAL. \nConfidence score is: {preds}"
 
     else:
-        preds = round(img_grads_np * 100, 3)
-        text2 = f"The image is FAKE."
+        preds = round(img_grads_np[1] * 100, 3)
+        text2 = f"The image is FAKE. \nConfidence score is: {preds}"
 
     return text2
 
@@ -182,7 +189,8 @@ def preprocess_video(input_video, n_frames = 3):
 def deepfakes_video_predict(input_video):
     '''Perform inference on a video.'''
     video_frames = preprocess_video(input_video)
-    grads_list = []
+    real_faces_list = []
+    fake_faces_list = []
 
     for face in video_frames:
         # face = preprocess_img(face)
@@ -190,15 +198,19 @@ def deepfakes_video_predict(input_video):
         img_grads = img_model.forward(face)
         img_grads = img_grads.cpu().detach().numpy()
         img_grads_np = np.squeeze(img_grads)
-        grads_list.append(img_grads_np)
+        real_faces_list.append(img_grads_np[0])
+        fake_faces_list.append(img_grads_np[1])
+
+    real_faces_mean = np.mean(real_faces_list)
+    fake_faces_mean = np.mean(fake_faces_list)
 
-    grads_list_mean = np.mean(grads_list)
+    if real_faces_mean > 0.5:
+        preds = round(real_faces_mean * 100, 3)
+        text2 = f"The video is REAL. \nConfidence score is: {preds}%"
 
-    if grads_list_mean > 0.5:
-        res = round(grads_list_mean * 100, 3)
-        text = f"The video is REAL."
     else:
-        res = round(100 - (grads_list_mean * 100), 3)
-        text = f"The video is FAKE."
-    return text
+        preds = round(fake_faces_mean * 100, 3)
+        text2 = f"The video is FAKE. \nConfidence score is: {preds}%"
+
+    return text2
 

checkpoints/model_best.pt → model.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:545246c0330351105d0197ce443163bbc35016167feff0afe64770670bb0ee09
-size 632423929
+oid sha256:7124691a506c8b7ea6baf3284a2b52f58879d3de89a8154d5ad3f69ba744912a
+size 152905281

requirements.txt

@@ -7,4 +7,6 @@ librosa
 ffmpeg
 albumentations
 opencv-python
-torchsummary
+torchsummary
+onnx
+onnx2pytorch

save_ckpts.py

@@ -0,0 +1,89 @@
+import onnx
+import torch
+import argparse
+import numpy as np
+import torch.nn as nn
+from models.TMC import ETMC
+from models import image
+from onnx2pytorch import ConvertModel
+
+onnx_model = onnx.load('checkpoints\\efficientnet.onnx')
+pytorch_model = ConvertModel(onnx_model)
+
+# Define the audio_args dictionary
+audio_args = {
+    'nb_samp': 64600,
+    'first_conv': 1024,
+    'in_channels': 1,
+    'filts': [20, [20, 20], [20, 128], [128, 128]],
+    'blocks': [2, 4],
+    'nb_fc_node': 1024,
+    'gru_node': 1024,
+    'nb_gru_layer': 3,
+    'nb_classes': 2
+}
+
+
+def get_args(parser):
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--data_dir", type=str, default="datasets/train/fakeavceleb*")
+    parser.add_argument("--LOAD_SIZE", type=int, default=256)
+    parser.add_argument("--FINE_SIZE", type=int, default=224)
+    parser.add_argument("--dropout", type=float, default=0.2)
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=1)
+    parser.add_argument("--hidden", nargs="*", type=int, default=[])
+    parser.add_argument("--hidden_sz", type=int, default=768)
+    parser.add_argument("--img_embed_pool_type", type=str, default="avg", choices=["max", "avg"])
+    parser.add_argument("--img_hidden_sz", type=int, default=1024)
+    parser.add_argument("--include_bn", type=int, default=True)
+    parser.add_argument("--lr", type=float, default=1e-4)
+    parser.add_argument("--lr_factor", type=float, default=0.3)
+    parser.add_argument("--lr_patience", type=int, default=10)
+    parser.add_argument("--max_epochs", type=int, default=500)
+    parser.add_argument("--n_workers", type=int, default=12)
+    parser.add_argument("--name", type=str, default="MMDF")
+    parser.add_argument("--num_image_embeds", type=int, default=1)
+    parser.add_argument("--patience", type=int, default=20)
+    parser.add_argument("--savedir", type=str, default="./savepath/")
+    parser.add_argument("--seed", type=int, default=1)
+    parser.add_argument("--n_classes", type=int, default=2)
+    parser.add_argument("--annealing_epoch", type=int, default=10)
+    parser.add_argument("--device", type=str, default='cpu')
+    parser.add_argument("--pretrained_image_encoder", type=bool, default = False)
+    parser.add_argument("--freeze_image_encoder", type=bool, default = False)
+    parser.add_argument("--pretrained_audio_encoder", type = bool, default=False)
+    parser.add_argument("--freeze_audio_encoder", type = bool, default = False)
+    parser.add_argument("--augment_dataset", type = bool, default = True)
+
+    for key, value in audio_args.items():
+        parser.add_argument(f"--{key}", type=type(value), default=value)
+
+def load_spec_modality_model(args):
+    spec_encoder = image.RawNet(args)
+    ckpt = torch.load('checkpoints\RawNet2.pth', map_location = torch.device('cpu'))
+    spec_encoder.load_state_dict(ckpt, strict = True)
+    spec_encoder.eval()
+    return spec_encoder
+
+
+#Load models.
+parser = argparse.ArgumentParser(description="Train Models")
+get_args(parser)
+args, remaining_args = parser.parse_known_args()
+assert remaining_args == [], remaining_args
+
+spec_model = load_spec_modality_model(args)
+
+print(f"Image model is: {pytorch_model}")
+
+print(f"Audio model is: {spec_model}")
+
+
+PATH = 'checkpoints\\model.pth'
+
+torch.save({
+    'spec_encoder': spec_model.state_dict(),
+    'rgb_encoder': pytorch_model.state_dict()
+}, PATH)
+
+print("Model saved.")