Initial commit

app.py

@@ -0,0 +1,7 @@
+import gradio as gr
+from whisperui import WhisperModelUI
+
+my_app = gr.Blocks()
+iface = WhisperModelUI(my_app)
+iface.create_whisper_ui()
+iface.launch()

requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/openai/whisper.git
+pytube

utils.py

@@ -0,0 +1,36 @@
+import whisper
+import os
+
+
+def whisper_decode(model, audio):
+    # model = whisper.load_model("base")
+
+    audio = whisper.pad_or_trim(audio)
+    # make log-Mel spectrogram and move to the same device as the model
+    mel = whisper.log_mel_spectrogram(audio).to(model.device)
+    # detect the spoken language
+    _, probs = model.detect_language(mel)
+    print(f"Detected language: {max(probs, key=probs.get)}")
+
+    # decode the audio
+    options = whisper.DecodingOptions(
+        task='translate',
+        fp16=False)
+    result = whisper.decode(model, mel, options)
+    # print the recognized text
+    print(result.text)
+
+
+def whisper_transcribe(model, audio):
+    result = model.transcribe(audio)
+    print(result["text"])
+
+
+def try_whisper_model(model_type, choice):
+    model = whisper.load_model(model_type)
+    data_file = os.path.join(os.path.curdir, 'data_files', 'bharat.mp3')
+    audio = whisper.load_audio(data_file)
+    if choice == 'decode':
+        whisper_decode(model, audio)
+    elif choice == 'transcribe':
+        whisper_transcribe(model, audio)

whisperui.py

@@ -0,0 +1,216 @@
+import whisper
+import gradio as gr
+import os
+from pytube import YouTube
+
+
+class WhisperModelUI(object):
+    def __init__(self, ui_obj):
+        self.name = "Whisper Model Processor UI"
+        self.description = "This class is designed to build UI for our Whisper Model"
+        self.ui_obj = ui_obj
+        self.audio_files_list = ['No content']
+        self.whisper_model = whisper.model.Whisper
+        self.video_store_path = 'data_files'
+
+    def load_content(self, file_list):
+        video_out_path = os.path.join(os.getcwd(), self.video_store_path)
+
+        self.audio_files_list = [f for f in os.listdir(video_out_path)
+                            if os.path.isfile(video_out_path + "/" + f)
+                            and (f.endswith(".mp4") or f.endswith('mp3'))]
+
+        return gr.Dropdown.update(choices=self.audio_files_list)
+
+    def load_whisper_model(self, model_type):
+        try:
+            asr_model = whisper.load_model(model_type.lower())
+            self.whisper_model = asr_model
+            status = "{} Model is loaded successfully".format(model_type)
+        except:
+            status = "error in loading {} model".format(model_type)
+
+        return status, str(self.whisper_model)
+
+    def load_youtube_video(self, video_url):
+        video_out_path = os.path.join(os.getcwd(), self.video_store_path)
+        yt = YouTube(video_url)
+        local_video_path = yt.streams.filter(progressive=True, file_extension='mp4').order_by(
+            'resolution').desc().first().download(video_out_path)
+        return local_video_path
+
+    def get_video_to_text(self,
+                          transcribe_or_decode,
+                          video_list_dropdown_file_name,
+                          language_detect,
+                          translate_or_transcribe
+                          ):
+        debug_text = ""
+        try:
+            video_out_path = os.path.join(os.getcwd(), 'data_files')
+            video_full_path = os.path.join(video_out_path, video_list_dropdown_file_name)
+            if not os.path.isfile(video_full_path):
+                video_text = "Selected video/audio is could not be located.."
+            else:
+                video_text = "Bad choice or result.."
+                if transcribe_or_decode == 'Transcribe':
+                    video_text, debug_text = self.run_asr_with_transcribe(video_full_path, language_detect,
+                                                                          translate_or_transcribe)
+                elif transcribe_or_decode == 'Decode':
+                    audio = whisper.load_audio(video_full_path)
+                    video_text, debug_text = self.run_asr_with_decode(audio, language_detect,
+                                                                      translate_or_transcribe)
+        except:
+            video_text = "Error processing audio..."
+        return video_text, debug_text
+
+    def run_asr_with_decode(self, audio, language_detect, translate_or_transcribe):
+        debug_info = "None.."
+
+        if 'encoder' not in dir(self.whisper_model) or 'decoder' not in dir(self.whisper_model):
+            return "Model is not loaded, please load the model first", debug_info
+
+        if self.whisper_model.encoder is None or self.whisper_model.decoder is None:
+            return "Model is not loaded, please load the model first", debug_info
+
+        try:
+            # pad/trim it to fit 30 seconds
+            audio = whisper.pad_or_trim(audio)
+
+            # make log-Mel spectrogram and move to the same device as the model
+            mel = whisper.log_mel_spectrogram(audio).to(self.whisper_model.device)
+
+            if language_detect == 'Detect':
+                # detect the spoken language
+                _, probs = self.whisper_model.detect_language(mel)
+                # print(f"Detected language: {max(probs, key=probs.get)}")
+
+            # decode the audio
+            # mps crash if fp16=False is not used
+
+            task_type = 'transcribe'
+            if translate_or_transcribe == 'Translate':
+                task_type = 'translate'
+
+            if language_detect != 'Detect':
+                options = whisper.DecodingOptions(fp16=False,
+                                                  language=language_detect,
+                                                  task=task_type)
+            else:
+                options = whisper.DecodingOptions(fp16=False,
+                                                  task=task_type)
+
+            result = whisper.decode(self.whisper_model, mel, options)
+            result_text = result.text
+            debug_info = str(result)
+        except:
+            result_text = "Error handing audio to text.."
+        return result_text, debug_info
+
+    def run_asr_with_transcribe(self, audio_path, language_detect, translate_or_transcribe):
+        result_text = "Error..."
+        debug_info = "None.."
+
+        if 'encoder' not in dir(self.whisper_model) or 'decoder' not in dir(self.whisper_model):
+            return "Model is not loaded, please load the model first", debug_info
+
+        if self.whisper_model.encoder is None or self.whisper_model.decoder is None:
+            return "Model is not loaded, please load the model first", debug_info
+
+        task_type = 'transcribe'
+        if translate_or_transcribe == 'Translate':
+            task_type = 'translate'
+
+        transcribe_options = dict(beam_size=5, best_of=5,
+                                  fp16=False,
+                                  task=task_type,
+                                  without_timestamps=False)
+        if language_detect != 'Detect':
+            transcribe_options['language'] = language_detect
+
+        transcription = self.whisper_model.transcribe(audio_path, **transcribe_options)
+        if transcription is not None:
+            result_text = transcription['text']
+            debug_info = str(transcription)
+        return result_text, debug_info
+
+    def create_whisper_ui(self):
+        with self.ui_obj:
+            gr.Markdown("AI翻訳・書き起こし")
+            with gr.Tabs():
+                with gr.TabItem("YouTubeURLから"):
+                    with gr.Row():
+                        with gr.Column():
+                            asr_model_type = gr.Radio(['Tiny', 'Base', 'Small', 'Medium', 'Large'],
+                                                      label="モデルタイプ（精度）",
+                                                      value='Base'
+                                                      )
+                            model_status_lbl = gr.Label(label="ローディングステータス")
+                            load_model_btn = gr.Button("モデルをロード")
+                            youtube_url = gr.Textbox(label="YouTube URL",
+                                                     # value="https://www.youtube.com/watch?v=Y2nHd7El8iw"
+                                                     value=""
+                                                     )
+                            youtube_video = gr.Video(label="ビデオ")
+                            get_video_btn = gr.Button("YouTubeURLをロード")
+                        with gr.Column():
+                            video_list_dropdown = gr.Dropdown(self.audio_files_list, label="保存済みビデオ")
+                            load_video_list_btn = gr.Button("全てのビデオをロード")
+                            transcribe_or_decode = gr.Radio(['Transcribe', 'Decode'],
+                                                            label="オプション（Transcribe = 書き起こし）",
+                                                            value='Transcribe'
+                                                            )
+                            language_detect = gr.Dropdown(['Detect', 'English', 'Hindi', 'Japanese'],
+                                                          label="自動検知か言語を選択")
+                            translate_or_transcribe = gr.Dropdown(['Transcribe', 'Translate'],
+                                                                  label="Translate（翻訳）か Transcribe（書き起こし）を選択")
+                            get_video_txt_btn = gr.Button("変換開始！")
+                            video_text = gr.Textbox(label="テキスト", lines=10)
+                with gr.TabItem("デバッグ情報"):
+                    with gr.Row():
+                        with gr.Column():
+                            debug_text = gr.Textbox(label="Debug Details", lines=20)
+            load_model_btn.click(
+                self.load_whisper_model,
+                [
+                    asr_model_type
+                ],
+                [
+                    model_status_lbl,
+                    debug_text
+                ]
+            )
+            get_video_btn.click(
+                self.load_youtube_video,
+                [
+                    youtube_url
+                ],
+                [
+                    youtube_video
+                ]
+            )
+            load_video_list_btn.click(
+                self.load_content,
+                [
+                    video_list_dropdown
+                ],
+                [
+                    video_list_dropdown
+                ]
+            )
+            get_video_txt_btn.click(
+                self.get_video_to_text,
+                [
+                    transcribe_or_decode,
+                    video_list_dropdown,
+                    language_detect,
+                    translate_or_transcribe
+                ],
+                [
+                    video_text,
+                    debug_text
+                ]
+            )
+
+    def launch_ui(self):
+        self.ui_obj.launch(debug=True)