solution 2

.gitattributes

@@ -1,5 +1,5 @@
 <<<<<<< HEAD
-checkpoints/model_best.pt filter=lfs diff=lfs merge=lfs -text
+checkpoints/model_best.pt filter=lfs diff=lfs merge=lfs -text
 =======
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
@@ -37,3 +37,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 >>>>>>> 10b34e9e01a793df83cca1499ece5c6b29f10a90
+checkpoints/model.pth filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-import inference
+import inference_2 as inference
 
 
 title="Multimodal deepfake detector"

checkpoints/model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd7e7092a26ba6b2927a05150d25f03fb19e4562006835cfa585a055b419f2f2
+size 604878654

inference_2.py

@@ -0,0 +1,217 @@
+import os
+import cv2
+import torch
+import argparse
+import numpy as np
+import torch.nn as nn
+from models.TMC import ETMC
+from models import image
+
+#Set random seed for reproducibility.
+torch.manual_seed(42)
+
+
+# Define the audio_args dictionary
+audio_args = {
+    'nb_samp': 64600,
+    'first_conv': 1024,
+    'in_channels': 1,
+    'filts': [20, [20, 20], [20, 128], [128, 128]],
+    'blocks': [2, 4],
+    'nb_fc_node': 1024,
+    'gru_node': 1024,
+    'nb_gru_layer': 3,
+    'nb_classes': 2
+}
+
+
+def get_args(parser):
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--data_dir", type=str, default="datasets/train/fakeavceleb*")
+    parser.add_argument("--LOAD_SIZE", type=int, default=256)
+    parser.add_argument("--FINE_SIZE", type=int, default=224)
+    parser.add_argument("--dropout", type=float, default=0.2)
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=1)
+    parser.add_argument("--hidden", nargs="*", type=int, default=[])
+    parser.add_argument("--hidden_sz", type=int, default=768)
+    parser.add_argument("--img_embed_pool_type", type=str, default="avg", choices=["max", "avg"])
+    parser.add_argument("--img_hidden_sz", type=int, default=1024)
+    parser.add_argument("--include_bn", type=int, default=True)
+    parser.add_argument("--lr", type=float, default=1e-4)
+    parser.add_argument("--lr_factor", type=float, default=0.3)
+    parser.add_argument("--lr_patience", type=int, default=10)
+    parser.add_argument("--max_epochs", type=int, default=500)
+    parser.add_argument("--n_workers", type=int, default=12)
+    parser.add_argument("--name", type=str, default="MMDF")
+    parser.add_argument("--num_image_embeds", type=int, default=1)
+    parser.add_argument("--patience", type=int, default=20)
+    parser.add_argument("--savedir", type=str, default="./savepath/")
+    parser.add_argument("--seed", type=int, default=1)
+    parser.add_argument("--n_classes", type=int, default=2)
+    parser.add_argument("--annealing_epoch", type=int, default=10)
+    parser.add_argument("--device", type=str, default='cpu')
+    parser.add_argument("--pretrained_image_encoder", type=bool, default = False)
+    parser.add_argument("--freeze_image_encoder", type=bool, default = False)
+    parser.add_argument("--pretrained_audio_encoder", type = bool, default=False)
+    parser.add_argument("--freeze_audio_encoder", type = bool, default = False)
+    parser.add_argument("--augment_dataset", type = bool, default = True)
+
+    for key, value in audio_args.items():
+        parser.add_argument(f"--{key}", type=type(value), default=value)
+
+def model_summary(args):
+    '''Prints the model summary.'''
+    model = ETMC(args)
+
+    for name, layer in model.named_modules():
+        print(name, layer)
+
+def load_multimodal_model(args):
+    '''Load multimodal model'''
+    model = ETMC(args)
+    ckpt = torch.load('checkpoints\\model.pth', map_location = torch.device('cpu'))
+    model.load_state_dict(ckpt,strict = True)
+    model.eval()
+    return model
+
+def load_img_modality_model(args):
+    '''Loads image modality model.'''
+    rgb_encoder = image.ImageEncoder(args)
+    ckpt = torch.load('checkpoints/model.pth', map_location = torch.device('cpu'))
+    rgb_encoder.load_state_dict(ckpt, strict = False)
+    rgb_encoder.eval()
+    return rgb_encoder
+
+def load_spec_modality_model(args):
+    spec_encoder = image.RawNet(args)
+    ckpt = torch.load('checkpoints/model.pth', map_location = torch.device('cpu'))
+    spec_encoder.load_state_dict(ckpt, strict = False)
+    spec_encoder.eval()
+    return spec_encoder
+
+
+#Load models.
+parser = argparse.ArgumentParser(description="Train Models")
+get_args(parser)
+args, remaining_args = parser.parse_known_args()
+assert remaining_args == [], remaining_args
+
+# multimodal = load_multimodal_model(args)
+spec_model = load_spec_modality_model(args)
+
+# print(f"Spec model is: {spec_model}")
+
+img_model = load_img_modality_model(args)
+
+# print(f"Image model is: {img_model}")
+
+# spec_in = torch.randn(1, 10_000)
+# rgb_in = torch.randn([1, 3, 256, 256])
+
+# rgb_out = img_model(rgb_in)
+# spec_out = spec_model(spec_in)
+
+# print(f"Img input shape is: {rgb_in.shape}, output shape: {rgb_out}")
+# print(f"Spec input shape is: {spec_in.shape}, output shape is: {spec_out.shape} output: {spec_out}")
+
+def preprocess_img(face):
+    face = face / 255
+    face = cv2.resize(face, (256, 256))
+    face = face.transpose(2, 0, 1) #(W, H, C) -> (C, W, H)
+    face_pt = torch.unsqueeze(torch.Tensor(face), dim = 0) 
+    return face_pt
+
+def preprocess_audio(audio_file):
+    audio_pt = torch.unsqueeze(torch.Tensor(audio_file), dim = 0)
+    return audio_pt
+
+def deepfakes_spec_predict(input_audio):
+    x, _ = input_audio
+    audio = preprocess_audio(x)
+    spec_grads = spec_model.forward(audio)
+    spec_grads_inv = np.exp(spec_grads.cpu().numpy().squeeze())
+
+    # multimodal_grads = multimodal.spec_depth[0].forward(spec_grads)
+
+    # out = nn.Softmax()(multimodal_grads)
+    # max = torch.argmax(out, dim = -1) #Index of the max value in the tensor.
+    # max_value = out[max] #Actual value of the tensor.
+    max_value = np.argmax(spec_grads_inv)
+
+    if max_value > 0.5:
+        preds = round(100 - (max_value*100), 3)
+        text2 = f"The audio is REAL."
+
+    else:
+        preds = round(max_value*100, 3)
+        text2 = f"The audio is FAKE."
+
+    return text2
+
+def deepfakes_image_predict(input_image):
+    face = preprocess_img(input_image)
+
+    img_grads = img_model.forward(face)
+    img_grads = img_grads.cpu().detach().numpy()
+    img_grads_np = np.squeeze(img_grads)
+
+    if img_grads_np > 0.5:
+        preds = round(100 - (img_grads_np * 100), 3)
+        text2 = f"The image is REAL."
+
+    else:
+        preds = round(img_grads_np * 100, 3)
+        text2 = f"The image is FAKE."
+
+    return text2
+
+
+def preprocess_video(input_video, n_frames = 3):
+    v_cap = cv2.VideoCapture(input_video)
+    v_len = int(v_cap.get(cv2.CAP_PROP_FRAME_COUNT))
+
+    # Pick 'n_frames' evenly spaced frames to sample
+    if n_frames is None:
+        sample = np.arange(0, v_len)
+    else:
+        sample = np.linspace(0, v_len - 1, n_frames).astype(int)
+
+    #Loop through frames.
+    frames = []
+    for j in range(v_len):
+        success = v_cap.grab()
+        if j in sample:
+            # Load frame
+            success, frame = v_cap.retrieve()
+            if not success:
+                continue
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            frame = preprocess_img(frame)
+            frames.append(frame)
+    v_cap.release()
+    return frames
+
+
+def deepfakes_video_predict(input_video):
+    '''Perform inference on a video.'''
+    video_frames = preprocess_video(input_video)
+    grads_list = []
+
+    for face in video_frames:
+        # face = preprocess_img(face)
+
+        img_grads = img_model.forward(face)
+        img_grads = img_grads.cpu().detach().numpy()
+        img_grads_np = np.squeeze(img_grads)
+        grads_list.append(img_grads_np)
+
+    grads_list_mean = np.mean(grads_list)
+
+    if grads_list_mean > 0.5:
+        res = round(grads_list_mean * 100, 3)
+        text = f"The video is REAL."
+    else:
+        res = round(100 - (grads_list_mean * 100), 3)
+        text = f"The video is FAKE."
+    return text
+

models/image.py

@@ -14,7 +14,8 @@ class ImageEncoder(nn.Module):
         self.device = args.device
         self.args = args
         self.flatten = nn.Flatten()
-        self.fc = nn.Linear(in_features=2560, out_features = 1024)
+        self.sigmoid = nn.Sigmoid()
+        # self.fc = nn.Linear(in_features=2560, out_features = 2)
         self.pretrained_image_encoder = args.pretrained_image_encoder
         self.freeze_image_encoder = args.freeze_image_encoder
 
@@ -22,24 +23,25 @@ class ImageEncoder(nn.Module):
             self.model = DeepFakeClassifier(encoder = "tf_efficientnet_b7_ns").to(self.device)
 
         else:
-            self.pretrained_ckpt = torch.load('DFDT TMC/pretrained/final_999_DeepFakeClassifier_tf_efficientnet_b7_ns_0_23', map_location = torch.device(self.args.device))
+            self.pretrained_ckpt = torch.load('pretrained\\final_999_DeepFakeClassifier_tf_efficientnet_b7_ns_0_23', map_location = torch.device(self.args.device))
             self.state_dict = self.pretrained_ckpt.get("state_dict", self.pretrained_ckpt)
 
             self.model = DeepFakeClassifier(encoder = "tf_efficientnet_b7_ns").to(self.device)
             print("Loading pretrained image encoder...")
-            self.model.load_state_dict({re.sub("^module.", "", k): v for k, v in self.state_dict.items()}, strict=False)
+            self.model.load_state_dict({re.sub("^module.", "", k): v for k, v in self.state_dict.items()}, strict=True)
             print("Loaded pretrained image encoder.")
 
         if self.freeze_image_encoder == True:
             for idx, param in self.model.named_parameters():
                 param.requires_grad = False
 
-        self.model.fc = nn.Identity()
+        # self.model.fc = nn.Identity()
 
     def forward(self, x):
         x = self.model(x)
-        x = self.flatten(x)
-        out = self.fc(x)
+        out = self.sigmoid(x)
+        # x = self.flatten(x)
+        # out = self.fc(x)
         return out
 
 
@@ -84,7 +86,12 @@ class RawNet(nn.Module):
 			hidden_size = args.gru_node,
 			num_layers = args.nb_gru_layer,
 			batch_first = True)
-			
+        
+        self.fc1_gru = nn.Linear(in_features = args.gru_node,
+			out_features = args.nb_fc_node)
+       
+        self.fc2_gru = nn.Linear(in_features = args.nb_fc_node,
+			out_features = args.nb_classes ,bias=True)
        
         self.sig = nn.Sigmoid()
         self.logsoftmax = nn.LogSoftmax(dim=1)
@@ -93,9 +100,9 @@ class RawNet(nn.Module):
 
         if self.pretrained_audio_encoder == True:
             print("Loading pretrained audio encoder")
-            ckpt = torch.load('DFDT TMC/pretrained/RawNet.pth', map_location = torch.device(self.device))
+            ckpt = torch.load('pretrained\\RawNet.pth', map_location = torch.device(self.device))
             print("Loaded pretrained audio encoder")
-            self.load_state_dict(ckpt, strict = False)
+            self.load_state_dict(ckpt, strict = True)
         
         if self.freeze_audio_encoder:
                 for param in self.parameters():
@@ -155,7 +162,10 @@ class RawNet(nn.Module):
         x = x.permute(0, 2, 1)     #(batch, filt, time) >> (batch, time, filt)
         self.gru.flatten_parameters()
         x, _ = self.gru(x)
-        output = x[:,-1,:]
+        x = x[:,-1,:]
+        x = self.fc1_gru(x)
+        x = self.fc2_gru(x)
+        output=self.logsoftmax(x)
       
         return output