Update app.py

app.py

@@ -2,6 +2,8 @@ import gradio as gr
 from transformers import pipeline, WhisperProcessor, WhisperForConditionalGeneration
 from diffusers import StableDiffusionPipeline
 import torch
+import numpy as np
+import soundfile as sf
 
 # Step 1: Prompt-to-Prompt Generation using BART (or any LLM except GPT or DeepSeek)
 prompt_generator = pipeline("text2text-generation", model="facebook/bart-large-cnn")
@@ -34,9 +36,12 @@ def generate_image(prompt: str, creativity: float, include_background: bool):
 processor = WhisperProcessor.from_pretrained("openai/whisper-large")
 model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large")
 
-def transcribe_audio(audio):
+def transcribe_audio(audio: np.ndarray, sampling_rate: int) -> str:
+    # Save the audio as a temporary WAV file
+    sf.write("temp_audio.wav", audio, sampling_rate)
+    
     # Convert audio to text using Whisper
-    audio_input = processor(audio, return_tensors="pt").input_features
+    audio_input = processor("temp_audio.wav", return_tensors="pt").input_features
     predicted_ids = model.generate(audio_input)
     transcription = processor.decode(predicted_ids[0], skip_special_tokens=True)
     return transcription
@@ -51,9 +56,9 @@ def process_input(description: str, creativity: float, include_background: bool)
     
     return prompt, image
 
-def process_audio_input(audio):
+def process_audio_input(audio, sampling_rate):
     # Convert audio to text
-    description = transcribe_audio(audio)
+    description = transcribe_audio(audio, sampling_rate)
     # Generate a prompt and image based on transcribed text
     prompt = generate_prompt(description)
     image = generate_image(prompt, creativity=0.7, include_background=True)
@@ -64,7 +69,7 @@ text_input = gr.Textbox(label="Enter Description", placeholder="E.g., A magical
 creativity_slider = gr.Slider(minimum=0, maximum=1, step=0.1, label="Creativity (0 to 1)", value=0.7)
 background_checkbox = gr.Checkbox(label="Include Background", value=True)
 
-audio_input = gr.Audio(type="numpy", label="Speak your Description")
+audio_input = gr.Audio(type="numpy", label="Speak your Description", source="microphone")
 
 # Create Gradio interface for text input
 interface = gr.Interface(
@@ -94,3 +99,4 @@ interface_with_audio = gr.Interface(
 
 # Launch the interface with multiple tabs for text and voice input
 gr.TabbedInterface([interface, interface_with_audio]).launch()
+