Add log_tools to use for implementing better logging to the Gradio console TODO at a future date

Print ongoing FFMPEG output to command line for debugging
Remove vcodec and acodec specifications for conversion of webm to !wav as it was taking ages to transcode opus to aac / vp9 --> h264

app.py

@@ -3,9 +3,7 @@ import re
 import threading
 import time
 from pathlib import Path
-
 import random
-
 import gradio as gr
 
 from utils import (HHMMSS_to_sec,  molly_xlsx_to_table, convert_and_trim_video,
@@ -43,12 +41,18 @@ def trim_video_helper(input_file, output_file, start_time, end_time):
     # Trim the video
     print("...start trimming")
     output_file = convert_and_trim_video(input_file.name, output_file, start_time, end_time)
+    if not output_file:
+        print("...trimming failed due to FFMPEG error")
+        return None
     print("...finished trimming")
     return output_file
 
 def convert_video_helper(input_file, output_file, output_format):
     # convert video
     output_file = convert_and_trim_video(input_file.name, output_file)
+    if not output_file:
+        print("...converting failed due to FFMPEG error") 
+        return None
     print("...finished converting")
     return output_file
 
@@ -78,21 +82,23 @@ def trim_video_vtr(input_file, output_format):
         output_file = set_output_file(input_file, output_format, output_folder, insert_string)
 
         # write the start time, and end time to a txt file
-        log_file = f"{os.getcwd()}/results/{audio_base_name}_start_end_time.txt"
-        print("log_file: ", log_file)
-        with open(log_file, "w") as f:
+        time_file = f"{os.getcwd()}/results/{audio_base_name}_start_end_time.txt"
+        print("time_file: ", time_file)
+        with open(time_file, "w") as f:
             f.write(f"{start_time}\n")
             f.write(f"{end_time}\n")
 
         # Trim the video
         output_file = trim_video_helper(input_file, output_file, start_time, end_time)
-
+        if not output_file:
+            gr.Error(f"Error: FFMPEG failed to trim the video.")
+            return None, None
         # delete threading
         print("Done trimming. Deleting files...")
-        path_to_delete = [input_file.name, output_file, log_file]
+        path_to_delete = [input_file.name, output_file, time_file]
         threading.Thread(target=delete_files, args=([path_to_delete])).start()
         
-        return output_file, log_file
+        return output_file, time_file
     except Exception as e:
         gr.Error(f"Error: {str(e)}")
         return f"Error: {e}"
@@ -109,7 +115,9 @@ def trim_video_wt(input_file, input_transcript, output_format, start_time, end_t
 
         # Trim the video
         output_file = trim_video_helper(input_file, output_file, start_time, end_time)
-
+        if not output_file:
+            gr.Error(f"Error: FFMPEG failed to trim the video.")
+            return None, None
         # convert transcript
         path = input_transcript.name
         output_transcript = convert_transcript_helper(path, output_transcript)
@@ -149,7 +157,9 @@ def trim_video(input_file, output_format, start_time, end_time):
 
         # Trim the video
         output_file = trim_video_helper(input_file, output_file, start_time, end_time)
-
+        if not output_file:
+            gr.Error(f"Error: FFMPEG failed to trim the video.")
+            return None
         # Remove files after 10 minutes for security
         print("Done trimming. Deleting files...")
         path_to_delete = [input_file.name, output_file]
@@ -169,6 +179,9 @@ def convert_video(input_file, output_format):
                                       insert_string = 'converted')
         # Convert video
         output_file = convert_video_helper(input_file, output_file, output_format)
+        if not output_file:
+            gr.Error(f"Error: FFMPEG failed to convert the video.")
+            return None
         print(f"...created output file: {output_file}")
         # remove file after 10 minutes for security 
         print("Done converting. Deleting files...")

log_tools.py

@@ -0,0 +1,58 @@
+# https://github.com/gradio-app/gradio/issues/2362
+import re
+import sys
+
+class Logger:
+    def __init__(self, filename):
+
+        self.filename = f"{LOG_DIR}{filename}"
+        self.terminal = sys.stdout
+        self.reset_logs()
+        self.log = open(self.filename, "w")
+        self.flush()
+
+    def write(self, message):
+        self.terminal.write(message)
+        self.log.write(message)
+
+    def flush(self):
+        self.terminal.flush()
+        self.log.flush()
+
+    def isatty(self):
+        return False
+
+    def reset_logs(self):
+        with open(self.filename, 'w') as file:
+            file.truncate(0)
+
+    def read_logs(self):
+        sys.stdout.flush()
+
+        # Read the entire content of the log file
+        with open(self.filename, "r") as f:
+            log_content = f.readlines()
+
+        # Filter out lines containing null characters
+        log_content = [line for line in log_content if '\x00' not in line]
+
+        # Define the regex pattern for the progress bar
+        progress_pattern = re.compile(r'\[.*\] \d+\.\d+%')
+
+        # Find lines matching the progress bar pattern
+        progress_lines = [line for line in log_content if
+                          progress_pattern.search(line) and " - Completed!\n" not in line]
+
+        # If there are multiple progress bars, keep only the last one in recent_lines
+        if progress_lines:
+            valid_content = [line for line in log_content if line not in progress_lines]
+            if log_content[-1] == progress_lines[-1]:
+                valid_content.append(progress_lines[-1].strip("\n"))
+        else:
+            valid_content = log_content
+
+        # Get the latest 30 lines
+        recent_lines = valid_content[-30:]
+
+        # Return the joined recent lines
+        return ''.join(recent_lines)

utils.py

@@ -4,7 +4,7 @@ import os
 import re
 import subprocess
 from pathlib import Path
-
+import sys
 import gradio as gr
 import pandas as pd
 
@@ -15,6 +15,11 @@ logging.basicConfig(filename=f'{os.getcwd()}/logs/logfile.log', level=logging.IN
                     format='%(asctime)s - %(levelname)s - %(message)s')
 logging.info('Starting the application...')
 
+
+def subprocess_run_verbose(cmd):
+    res = subprocess.check_call(cmd, stdout=sys.stdout, stderr=subprocess.STDOUT)
+    return res
+
 def sort_transcript(file_path: str, save_path: str) -> str:
     """
     Sort the rows of a transcript file by start time.
@@ -216,8 +221,12 @@ def convert_and_trim_video(media_in, media_out, start=None, end=None):
 
         if out_ext == '.wav':
             if in_ext == '.webm':
-                command = ['ffmpeg', '-i', media_in, *trim_command, media_out, 
-                    '-hide_banner', '-loglevel', 'warning']
+                command = [
+                    'ffmpeg', '-y',
+                    '-i', media_in, 
+                    *trim_command, 
+                    media_out, 
+                    '-hide_banner', '-loglevel', 'info']
 
             else: 
                 # convert to wav with standard format for audio models
@@ -232,38 +241,53 @@ def convert_and_trim_video(media_in, media_out, start=None, end=None):
                     '-ac', str(WAV_CHANNELS), 
                     '-ar', str(WAV_SAMPLE_RATE), 
                     media_out, 
-                    '-hide_banner', '-loglevel', 'warning']
+                    '-hide_banner', '-loglevel', 'info']
 
         else: # convert using copy codec 
             if in_ext == '.webm':
-                command = ['ffmpeg', 
-                '-i', media_in,
-                *trim_command,
-                '-c', 'copy', 
-                '-vcodec', 'h264', 
-                '-acodec', 'aac', 
-                media_out,
-                '-hide_banner', '-loglevel', 'warning']
+                command = [
+                    'ffmpeg', '-y',
+                    '-i', media_in,
+                    '-strict', '-2',
+                    *trim_command,
+                    '-c', 'copy', 
+                    # '-vcodec', 'h264', 
+                    # '-acodec', 'aac', 
+                    media_out,
+                    '-hide_banner', '-loglevel', 'info']
             else: # not webm
-                command = ['ffmpeg',
-                                '-y',
-                                '-i', media_in,
-                                *trim_command,
-                                '-c','copy',
-                                media_out,
-                                '-hide_banner', '-loglevel', 'warning']
+                command = [
+                    'ffmpeg',
+                    '-y',
+                    '-i', media_in,
+                    *trim_command,
+                    '-c','copy',
+                    media_out,
+                    '-hide_banner', '-loglevel', 'info']
 
         # run the ffmpeg command
         logging.info(f"FFMPEG command: {' '.join(command)}")
         gr.Info(f"FFMPEG command: {' '.join(command)}", visible=False)
-        process = subprocess.run(command, capture_output=True, text=True)
-        if process.returncode != 0:
-            logging.info(f"FFMPEG error: {process.stderr}")
-            print(f"FFMPEG error: {process.stderr}")
-            gr.Error(f"FFMPEG error: {process.stderr}")
+        print(f"...FFMPEG command: {' '.join(command)}")
+        # process = subprocess.run(command, capture_output=True, text=True)
+        # if process.returncode != 0:
+        #     logging.info(f"FFMPEG error: {process.stderr}")
+        #     print(f"FFMPEG error: {process.stderr}")
+        #     gr.Error(f"FFMPEG error: {process.stderr}")
+        # else:
+        #     logging.info(process.stdout)
+        #     print(f"...FFMPEG status: {process.stdout}")
+        return_code = subprocess_run_verbose(command)
+        print(f"FFMPEG return code: {return_code}")
+        if return_code != 0:
+            logging.info(f"FFMPEG error: {return_code}")
+            print(f"FFMPEG error: {return_code}")
+            gr.Error(f"FFMPEG error: {return_code}")
+            return None
         else:
-            logging.info(process.stdout)
-        return media_out
+            logging.info(f"...FFMPEG completed successfully...")
+            print(f"...FFMPEG completed successfully...")
+            return media_out
 
     except Exception as e:
         print(f"Error converting video format: {e}")