Made changes for audio

app.py

@@ -1,7 +1,7 @@
 import torch
 import torch.nn as nn
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
 
 
 class _MLPVectorProjector(nn.Module):
@@ -20,10 +20,35 @@ class _MLPVectorProjector(nn.Module):
     def forward(self, x):
         return torch.cat([mlp(x) for mlp in self.mlps], dim=-2)
 
+## Text model
+
 model_name = "microsoft/phi-2"
 
 phi2_text = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+tokenizer_text = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+## Audio model
+model_id_audio = "openai/whisper-large-v3"
+
+model_audio = AutoModelForSpeechSeq2Seq.from_pretrained(
+    model_id_audio, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True).to("cpu")
+
+processor_audio = AutoProcessor.from_pretrained(model_id_audio)
+
+pipe_audio = pipeline(
+    "automatic-speech-recognition",
+    model=model_audio,
+    tokenizer=processor_audio.tokenizer,
+    feature_extractor=processor_audio.feature_extractor,
+    max_new_tokens=128,
+    chunk_length_s=30,
+    batch_size=16,
+    return_timestamps=False,
+    torch_dtype=torch.float32,
+    device="cpu",
+)
+
+## image model
 
 def example_inference(input_text, count): #, image, img_qn, audio):
     pred_text = textMode(input_text, count)
@@ -33,8 +58,8 @@ def example_inference(input_text, count): #, image, img_qn, audio):
 
 def textMode(text, count):
     count = int(count)
-    inputs = tokenizer(text, return_tensors="pt", return_attention_mask=False)
-    prediction = tokenizer.batch_decode(
+    inputs = tokenizer_text(text, return_tensors="pt", return_attention_mask=False)
+    prediction = tokenizer_text.batch_decode(
     phi2_text.generate(
         **inputs, 
         max_new_tokens=count,
@@ -51,7 +76,8 @@ def imageMode(image, question):
     return "In progress"
 
 def audioMode(audio):
-    return "In progress"
+    result = pipe_audio(audio)
+    return result["text"]
 
 
 interface_title = "TSAI-ERA-V1 - Capstone - Multimodal GPT Demo"