Update app.py

app.py

@@ -2,11 +2,13 @@ import cv2
 import gradio as gr
 import tempfile
 import torch
+import torchaudio
 from torchvision.models.detection import fasterrcnn_resnet50_fpn
 import torchvision.transforms as transforms
 from PIL import Image
 import numpy as np
 import soundfile as sf
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
 
 class FasterRCNNDetector:
     def __init__(self):
@@ -49,8 +51,8 @@ class FasterRCNNDetector:
 class JarvisModels:
     def __init__(self):
         self.client1 = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
-        self.model = deepspeech.Model("deepspeech-0.9.3-models.pbmm")
-        self.model.setBeamWidth(500)
+        self.processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h")
+        self.model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h")
 
     async def generate_response(self, prompt):
         generate_kwargs = dict(
@@ -74,20 +76,18 @@ class JarvisModels:
             communicate.save(tmp_path)
         return tmp_path
 
-def transcribe_audio(audio_file):
-    model = JarvisModels().model
-    audio, sample_rate = sf.read(audio_file)
-    return model.stt(audio)
-
-def generate_response(frame):
-    jarvis = JarvisModels()
-    response_model = await jarvis.generate_response("Hello, I see some interesting objects!")
-    return response_model
+    async def transcribe_audio(self, audio_file):
+        input_audio, _ = torchaudio.load(audio_file)
+        input_values = self.processor(input_audio, return_tensors="pt").input_values
+        logits = self.model(input_values).logits
+        predicted_ids = torch.argmax(logits, dim=-1)
+        transcription = self.processor.batch_decode(predicted_ids)
+        return transcription[0]
 
 detector = FasterRCNNDetector()
 
 iface = gr.Interface(
-    fn=[detector.detect_objects, transcribe_audio],
+    fn=[detector.detect_objects, JarvisModels().transcribe_audio],
     inputs=gr.inputs.Video(label="Webcam", parameters={"fps": 30}),
     outputs=[gr.outputs.Image(), "text"],
     title="Vision and Speech Interface",