Update app.py

app.py

@@ -1,5 +1,6 @@
 import gradio as gr
 from PIL import Image, ImageDraw, ImageFont
+import scipy.io.wavfile as wavfile
 
 
 # Use a pipeline as a high-level helper
@@ -8,11 +9,58 @@ from transformers import pipeline
 # model_path = ("../Models/models--facebook--detr-resnet-50/snapshots"
 #               "/1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b")
 
+# object_detector = pipeline("object-detection",
+#                 model=model_path)
+
 object_detector = pipeline("object-detection",
                 model="facebook/detr-resnet-50")
 
-# object_detector = pipeline("object-detection",
-#                 model=model_path)
+narrator = pipeline("text-to-speech",
+                    model="kakao-enterprise/vits-ljs")
+
+
+# Define the function to generate audio from text
+def generate_audio(text):
+    # Generate the narrated text
+    narrated_text = narrator(text)
+
+    # Save the audio to a WAV file
+    wavfile.write("output.wav", rate=narrated_text["sampling_rate"],
+                  data=narrated_text["audio"][0])
+
+    # Return the path to the saved audio file
+    return "output.wav"
+
+
+
+def read_objects(detection_objects):
+    # Initialize counters for each object label
+    object_counts = {}
+
+    # Count the occurrences of each label
+    for detection in detection_objects:
+        label = detection['label']
+        if label in object_counts:
+            object_counts[label] += 1
+        else:
+            object_counts[label] = 1
+
+    # Generate the response string
+    response = "This picture contains"
+    labels = list(object_counts.keys())
+    for i, label in enumerate(labels):
+        response += f" {object_counts[label]} {label}"
+        if object_counts[label] > 1:
+            response += "s"
+        if i < len(labels) - 2:
+            response += ","
+        elif i == len(labels) - 2:
+            response += " and"
+
+    response += "."
+
+    return response
+
 
 
 def draw_bounding_boxes(image, detections, font_path=None, font_size=50):
@@ -61,7 +109,10 @@ def detect_object(image):
     raw_image = image
     output = object_detector(raw_image)
     processed_image = draw_bounding_boxes(raw_image, output)
-    return processed_image
+    natural_text = read_objects(output)
+    processed_audio = generate_audio(natural_text)
+    return processed_image, processed_audio
+    
 
 examples = [
     ["example1.jpg"],
@@ -74,7 +125,7 @@ demo = gr.Interface(fn=detect_object,
                     outputs=[gr.Image(label="Processed Image", type="pil")],
                     examples = examples,
                     title="Object Detector",
-                    description="Detect objects in the input image with bounding boxes.")
+                    description="Detect objects in the input image with bounding boxes with audio description.")
 demo.launch()