Update app.py

app.py

@@ -70,12 +70,15 @@ import gradio as gr
 from transformers import BlipProcessor, BlipForConditionalGeneration, pipeline
 from PIL import Image
 import torch
-from transformers import pipeline
+from gtts import gTTS
+import tempfile
+import os
+
 # ----------------------
-# Load BLIP (Large) for Captioning + VQA
+# Load BLIP (Large) for Captioning
 # ----------------------
-processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
-model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
+caption_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
+caption_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
 
 # ----------------------
 # Translation pipelines
@@ -87,61 +90,57 @@ translation_models = {
 }
 
 # ----------------------
-# Caption + Translate Function
+# Caption + Translate + Speak Function
 # ----------------------
-def generate_caption_translate(image, target_lang):
-    inputs = processor(images=image, return_tensors="pt")
-    out = model.generate(**inputs, max_new_tokens=50)
-    english_caption = processor.decode(out[0], skip_special_tokens=True)
+def generate_caption_translate_speak(image, target_lang):
+    # Step 1: Caption
+    inputs = caption_processor(images=image, return_tensors="pt")
+    out = caption_model.generate(**inputs, max_new_tokens=50)
+    english_caption = caption_processor.decode(out[0], skip_special_tokens=True)
 
-    # Translate if chosen
+    # Step 2: Translate
     if target_lang in translation_models:
         translated = translation_models[target_lang](english_caption)[0]['translation_text']
     else:
         translated = "Translation not available"
 
-    return english_caption, translated
+    # Step 3: Generate Speech (English caption for now)
+    tts = gTTS(english_caption, lang="en")
+    tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
+    tts.save(tmp_file.name)
+    audio_file = tmp_file.name
+
+    return english_caption, translated, audio_file
 
 # ----------------------
-# VQA Function (using same BLIP model)
+# VQA Function (using BLIP VQA)
 # ----------------------
-
-
 from transformers import BlipProcessor, BlipForQuestionAnswering
-from PIL import Image
-import torch
 
-# Load BLIP VQA
-processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
-model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to("cuda" if torch.cuda.is_available() else "cpu")
+vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to("cuda" if torch.cuda.is_available() else "cpu")
 
-# Function
 def vqa_answer(image, question):
-    # image is already a PIL Image (no need to open again)
-    inputs = processor(image, question, return_tensors="pt").to(model.device)
-    out = model.generate(**inputs, max_new_tokens=50)
-    answer = processor.decode(out[0], skip_special_tokens=True)
+    inputs = vqa_processor(image, question, return_tensors="pt").to(vqa_model.device)
+    out = vqa_model.generate(**inputs, max_new_tokens=50)
+    answer = vqa_processor.decode(out[0], skip_special_tokens=True)
     return answer
 
-
-# Example
-# print(vqa_answer("baby.jpg", "What is the baby eating?"))
-
-
 # ----------------------
 # Gradio UI
 # ----------------------
 with gr.Blocks(title="BLIP Vision App") as demo:
-    gr.Markdown("## 🖼️ BLIP: Image Captioning + Translation + Question Answering")
+    gr.Markdown("## 🖼️ BLIP: Image Captioning + Translation + Speech + VQA")
 
-    with gr.Tab("Caption + Translate"):
+    with gr.Tab("Caption + Translate + Speak"):
         with gr.Row():
             img_in = gr.Image(type="pil", label="Upload Image")
             lang_in = gr.Dropdown(["Hindi", "French", "Spanish"], label="Translate To")
         eng_out = gr.Textbox(label="English Caption")
         trans_out = gr.Textbox(label="Translated Caption")
-        btn1 = gr.Button("Generate Caption & Translate")
-        btn1.click(generate_caption_translate, inputs=[img_in, lang_in], outputs=[eng_out, trans_out])
+        audio_out = gr.Audio(label="Spoken Caption")
+        btn1 = gr.Button("Generate Caption, Translate & Speak")
+        btn1.click(generate_caption_translate_speak, inputs=[img_in, lang_in], outputs=[eng_out, trans_out, audio_out])
 
     with gr.Tab("Visual Question Answering (VQA)"):
         with gr.Row():
@@ -152,3 +151,6 @@ with gr.Blocks(title="BLIP Vision App") as demo:
         btn2.click(vqa_answer, inputs=[img_vqa, q_in], outputs=ans_out)
 
 demo.launch()
+
+       
+