Update app.py

app.py

@@ -1,43 +1,46 @@
 import os
-os.system("pip install scipy")
-# os.system('pip install -r requirements.txt')
+# os.system("pip install scipy")
+os.system('pip install -r requirements.txt')
 
-from PIL import Image
-import io
 import streamlit as st
+from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
+from datasets import load_dataset
+import torch
+import soundfile as sf
 from transformers import pipeline
-import scipy
-
-# 初始化视觉问题回答和文本到语音的管道
-vqa_pipeline = pipeline("visual-question-answering", model="microsoft/git-base-vqav2")
-tts_pipeline = pipeline("text-to-speech", "suno/bark")
-
-def main():
-    st.title("Visual Question Answering & Text-to-Audio App")
-    
-    image = st.file_uploader("Upload an image", type=["jpg", "png"])
-    question = st.text_input("Enter your question")
-
-    if st.button('Run Visual Question Answering'):
-        if image and question:
-            # 将上传的文件转换为 PIL 图片
-            image = Image.open(io.BytesIO(image.getvalue()))
-
-            # 对用户上传的图片和问题进行视觉问题回答
-            vqa_result = vqa_pipeline({"image": image, "question": question})
-
-            # 这里假设vqa_result返回的是一个列表，其中包含字典，且字典里有'answer'键
-            if vqa_result:  # 确保vqa_result不为空
-                answer = vqa_result[0]['answer']  # 获取回答
-                st.write(f"Answer: {answer}")  # 显示回答
-
-                # 将回答转换为音频并播放
-                speech = tts_pipeline(answer, forward_params={"do_sample": True})
-                scipy.io.wavfile.write("bark_out.wav", rate=speech["sampling_rate"], data=speech["audio"])
-                
-                st.audio("bark_out.wav", format="audio/wav")
-        else:
-            st.write("Please input an image and a question first.")
-
-if __name__ == "__main__":
-    main()
+from PIL import Image
+import io
+
+st.title('Video to text and then text to speech app')
+
+
+image = st.file_uploader("Upload an image", type=["jpg", "png"])
+
+question = st.text_input(
+    label="Enter your question",
+	value = "How many people and what is the color of this image?"
+)
+
+def generate_speech(text):
+    processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
+    model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
+    vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
+    inputs = processor(text=text, return_tensors="pt")
+
+    embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+    speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
+
+    speech = model.generate_speech(inputs["input_ids"], speaker_embeddings, vocoder=vocoder)
+
+    sf.write("speech.wav", speech.numpy(), samplerate=16000)
+
+if st.button("Generate"):
+    image = Image.open(io.BytesIO(image.getvalue()))
+    vqa_pipeline = pipeline("visual-question-answering", model="dandelin/vilt-b32-finetuned-vqa")
+    vqa_result = vqa_pipeline({"image": image, "question": question})
+    answer = vqa_result[0]['answer']
+    st.write(f"Question: {question} Answer: {answer}")  # 显示回答
+    generate_speech(f"Question: {question}, Answer: {answer}")
+    audio_file = open("speech.wav", 'rb')
+    audio_bytes = audio_file.read()
+    st.audio(audio_bytes, format="audio/wav")