Update app.py

app.py

@@ -1,85 +1,13 @@
 import cv2
 import gradio as gr
-import edge_tts
 import tempfile
-import numpy as np
+import torch
 from torchvision.models.detection import fasterrcnn_resnet50_fpn
 import torchvision.transforms as transforms
 from PIL import Image
-from huggingface_hub import InferenceClient
-
-class YoloDetector:
-    def __init__(self, weights_path, cfg_path, names_path):
-        self.net = cv2.dnn.readNet(weights_path, cfg_path)
-        self.classes = []
-        with open(names_path, "r") as f:
-            self.classes = [line.strip() for line in f.readlines()]
-        self.layer_names = self.net.getLayerNames()
-        self.output_layers = [self.layer_names[i[0] - 1] for i in self.net.getUnconnectedOutLayers()]
-    
-    def detect_objects(self, frame):
-        height, width, channels = frame.shape
-        blob = cv2.dnn.blobFromImage(frame, 0.00392, (416, 416), (0, 0, 0), True, crop=False)
-        self.net.setInput(blob)
-        outs = self.net.forward(self.output_layers)
-
-        class_ids = []
-        confidences = []
-        boxes = []
-        for out in outs:
-            for detection in out:
-                scores = detection[5:]
-                class_id = np.argmax(scores)
-                confidence = scores[class_id]
-                if confidence > 0.5:
-                    center_x = int(detection[0] * width)
-                    center_y = int(detection[1] * height)
-                    w = int(detection[2] * width)
-                    h = int(detection[3] * height)
-                    x = int(center_x - w / 2)
-                    y = int(center_y - h / 2)
-                    boxes.append([x, y, w, h])
-                    confidences.append(float(confidence))
-                    class_ids.append(class_id)
-
-        indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
-        font = cv2.FONT_HERSHEY_PLAIN
-        for i in range(len(boxes)):
-            if i in indexes:
-                x, y, w, h = boxes[i]
-                label = str(self.classes[class_ids[i]])
-                color = (0, 255, 0)
-                cv2.rectangle(frame, (x, y), (x + w, y + h), color, 2)
-                cv2.putText(frame, label, (x, y + 30), font, 3, color, 2)
-
-        return frame
-
-class JarvisModels:
-    def __init__(self):
-        self.client1 = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
-        self.detector = YoloDetector("yolov3.weights", "yolov3.cfg", "coco.names")
-    
-    async def generate_model1(self, prompt):
-        generate_kwargs = dict(
-            temperature=0.6,
-            max_new_tokens=256,
-            top_p=0.95,
-            repetition_penalty=1,
-            do_sample=True,
-            seed=42,
-        )
-        formatted_prompt = system_instructions1 + prompt + "[JARVIS]"
-        stream = self.client1.text_generation(
-            formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True)
-        output = ""
-        for response in stream:
-            output += response.token.text
-
-        communicate = edge_tts.Communicate(output)
-        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
-            tmp_path = tmp_file.name
-            communicate.save(tmp_path)
-        return tmp_path
+import deepspeech
+import numpy as np
+import soundfile as sf
 
 class FasterRCNNDetector:
     def __init__(self):
@@ -119,16 +47,51 @@ class FasterRCNNDetector:
         
         return image
 
+class JarvisModels:
+    def __init__(self):
+        self.client1 = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
+        self.model = deepspeech.Model("deepspeech-0.9.3-models.pbmm")
+        self.model.setBeamWidth(500)
+
+    async def generate_response(self, prompt):
+        generate_kwargs = dict(
+            temperature=0.6,
+            max_new_tokens=256,
+            top_p=0.95,
+            repetition_penalty=1,
+            do_sample=True,
+            seed=42,
+        )
+        formatted_prompt = system_instructions1 + prompt + "[JARVIS]"
+        stream = self.client1.text_generation(
+            formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True)
+        output = ""
+        for response in stream:
+            output += response.token.text
+
+        communicate = edge_tts.Communicate(output)
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
+            tmp_path = tmp_file.name
+            communicate.save(tmp_path)
+        return tmp_path
+
+def transcribe_audio(audio_file):
+    model = JarvisModels().model
+    audio, sample_rate = sf.read(audio_file)
+    return model.stt(audio)
+
 def generate_response(frame):
     jarvis = JarvisModels()
-    detector = FasterRCNNDetector()
-    frame_with_boxes = jarvis.detector.detect_objects(frame)
-    cv2.imwrite("temp.jpg", frame_with_boxes)
-    communicate = edge_tts.Communicate("Objects detected!")
-    with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
-        tmp_path = tmp_file.name
-        communicate.save(tmp_path)
-    return tmp_path
+    response_model = await jarvis.generate_response("Hello, I see some interesting objects!")
+    return response_model
+
+detector = FasterRCNNDetector()
 
-iface = gr.Webcam(gr.Video(label="Webcam", parameters=["fps=30"], is_streaming=True), preprocess=generate_response, postprocess=FasterRCNNDetector().detect_objects, show_loading=False)
-gr.Interface(fn=iface, layout="vertical", capture_session=True).launch()
+iface = gr.Interface(
+    fn=[detector.detect_objects, transcribe_audio],
+    inputs=gr.inputs.Video(label="Webcam", parameters={"fps": 30}),
+    outputs=[gr.outputs.Image(), "text"],
+    title="Vision and Speech Interface",
+    description="This interface detects objects in the webcam feed and transcribes speech recorded through the microphone."
+)
+iface.launch()