Update code for train large data

00-convert_video_to_image.py

@@ -1,28 +1,33 @@
-import json
+import csv
 import os
 import cv2
 import math
 
 base_path = '.\\train_sample_videos\\'
+videos_path = os.path.join(base_path, 'Deepfakes')
 
 def get_filename_only(file_path):
     file_basename = os.path.basename(file_path)
     filename_only = file_basename.split('.')[0]
     return filename_only
 
-with open(os.path.join(base_path, 'metadata.json')) as metadata_json:
-    metadata = json.load(metadata_json)
+with open(os.path.join(base_path, 'csv', 'Deepfakes.csv'), newline='', encoding='utf-8') as csvfile:
+    reader = csv.DictReader(csvfile)
+    metadata = {}
+    for row in reader:
+        metadata[row['File Path']] = row['Label'].strip().upper()
     print(len(metadata))
 
 for filename in metadata.keys():
-    print(filename)
-    if (filename.endswith(".mp4")):
-        tmp_path = os.path.join(base_path, get_filename_only(filename))
+    video_basename = os.path.basename(filename)
+    print(video_basename)
+    if (video_basename.endswith(".mp4")):
+        tmp_path = os.path.join(videos_path, get_filename_only(video_basename))
         print('Creating Directory: ' + tmp_path)
         os.makedirs(tmp_path, exist_ok=True)
         print('Converting Video to Images...')
         count = 0
-        video_file = os.path.join(base_path, filename)
+        video_file = os.path.join(videos_path, video_basename)
         cap = cv2.VideoCapture(video_file)
         frame_rate = cap.get(5) #frame rate
         while(cap.isOpened()):

01a-crop_faces_with_mtcnn.py

@@ -1,7 +1,7 @@
 import cv2
 from mtcnn import MTCNN
+import csv
 import sys, os.path
-import json
 from keras import backend as K
 import tensorflow as tf
 print(tf.__version__)
@@ -13,21 +13,29 @@ if physical_devices:
     tf.config.experimental.set_memory_growth(physical_devices[0], True)
 
 base_path = '.\\train_sample_videos\\'
+videos_path = os.path.join(base_path, 'Deepfakes')
 
 def get_filename_only(file_path):
     file_basename = os.path.basename(file_path)
     filename_only = file_basename.split('.')[0]
     return filename_only
 
-with open(os.path.join(base_path, 'metadata.json')) as metadata_json:
-    metadata = json.load(metadata_json)
+with open(os.path.join(base_path, 'csv', 'Deepfakes.csv'), newline='', encoding='utf-8') as csvfile:
+    reader = csv.DictReader(csvfile)
+    metadata = {}
+    for row in reader:
+        metadata[row['File Path']] = row['Label'].strip().upper()
     print(len(metadata))
 
 for filename in metadata.keys():
-    tmp_path = os.path.join(base_path, get_filename_only(filename))
+    video_basename = os.path.basename(filename)
+    tmp_path = os.path.join(videos_path, get_filename_only(video_basename))
     print('Processing Directory: ' + tmp_path)
-    frame_images = [x for x in os.listdir(tmp_path) if os.path.isfile(os.path.join(tmp_path, x))]
     faces_path = os.path.join(tmp_path, 'faces')
+    if os.path.isdir(faces_path) and len(os.listdir(faces_path)) > 0:
+        print('Skipping (faces already exist): ' + faces_path)
+        continue
+    frame_images = [x for x in os.listdir(tmp_path) if os.path.isfile(os.path.join(tmp_path, x))]
     print('Creating Directory: ' + faces_path)
     os.makedirs(faces_path, exist_ok=True)
     print('Cropping Faces from Images...')

02-prepare_fake_real_dataset.py

@@ -1,11 +1,10 @@
-import json
+import csv
 import os
-from distutils.dir_util import copy_tree
 import shutil
 import numpy as np
 import splitfolders as split_folders
 
-base_path = '.\\train_sample_videos\\'
+base_path = '.\\train_sample_videos\\Deepfakes\\'
 dataset_path = '.\\prepared_dataset\\'
 print('Creating Directory: ' + dataset_path)
 os.makedirs(dataset_path, exist_ok=True)
@@ -19,8 +18,11 @@ def get_filename_only(file_path):
     filename_only = file_basename.split('.')[0]
     return filename_only
 
-with open(os.path.join(base_path, 'metadata.json')) as metadata_json:
-    metadata = json.load(metadata_json)
+with open(os.path.join('.\\train_sample_videos\\csv\\', 'Deepfakes.csv'), newline='', encoding='utf-8') as csvfile:
+    reader = csv.DictReader(csvfile)
+    metadata = {}
+    for row in reader:
+        metadata[row['File Path']] = row['Label'].strip().upper()
     print(len(metadata))
 
 real_path = os.path.join(dataset_path, 'real')
@@ -31,18 +33,18 @@ fake_path = os.path.join(dataset_path, 'fake')
 print('Creating Directory: ' + fake_path)
 os.makedirs(fake_path, exist_ok=True)
 
-for filename in metadata.keys():
+for filename, label in metadata.items():
     print(filename)
-    print(metadata[filename]['label'])
+    print(label)
     tmp_path = os.path.join(os.path.join(base_path, get_filename_only(filename)), 'faces')
     print(tmp_path)
     if os.path.exists(tmp_path):
-        if metadata[filename]['label'] == 'REAL':    
+        if label == 'REAL':
             print('Copying to :' + real_path)
-            copy_tree(tmp_path, real_path)
-        elif metadata[filename]['label'] == 'FAKE':
+            shutil.copytree(tmp_path, real_path, dirs_exist_ok=True)
+        elif label == 'FAKE':
             print('Copying to :' + tmp_fake_path)
-            copy_tree(tmp_path, tmp_fake_path)
+            shutil.copytree(tmp_path, tmp_fake_path, dirs_exist_ok=True)
         else:
             print('Ignored..')
 

03-train_cnn.py

@@ -142,30 +142,6 @@ history = model.fit(
 )
 print(history.history)
 
-'''
-# Plot results
-import matplotlib.pyplot as plt
-
-acc = history.history['acc']
-val_acc = history.history['val_acc']
-loss = history.history['loss']
-val_loss = history.history['val_loss']
-
-epochs = range(1, len(acc) + 1)
-
-plt.plot(epochs, acc, 'bo', label = 'Training Accuracy')
-plt.plot(epochs, val_acc, 'b', label = 'Validation Accuracy')
-plt.title('Training and Validation Accuracy')
-plt.legend()
-plt.figure()
-
-plt.plot(epochs, loss, 'bo', label = 'Training loss')
-plt.plot(epochs, val_loss, 'b', label = 'Validation Loss')
-plt.title('Training and Validation Loss')
-plt.legend()
-
-plt.show()
-'''
 
 # load the saved model that is considered the best
 best_model = load_model(os.path.join(checkpoint_filepath, 'best_model.h5'))