Spaces:
Sleeping
Sleeping
File size: 3,867 Bytes
4eb567d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 |
import gradio as gr
from transformers import pipeline
import torch
from diffusers import DiffusionPipeline
# Load speech-to-text model (Whisper)
transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-base")
# Load image generation model (Stable Diffusion)
device = "cuda" if torch.cuda.is_available() else "cpu"
pipe = DiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16 if device == "cuda" else torch.float32
)
pipe = pipe.to(device)
# Speech-to-text function
def transcribe_audio(audio):
"""Convert audio to text using Whisper"""
if audio is None:
return ""
try:
# Gradio Audio with type="numpy" returns tuple of (sample_rate, audio_data)
if isinstance(audio, tuple):
sample_rate, audio_data = audio
# Create a dictionary with the audio data for the pipeline
result = transcriber({"array": audio_data, "sampling_rate": sample_rate})
else:
result = transcriber(audio)
text = result.get("text", "").strip()
return text if text else "No speech detected"
except Exception as e:
return f"Error transcribing audio: {str(e)}"
# Image generation function
def generate_image_from_text(prompt):
"""Generate an image from a text prompt using Stable Diffusion"""
if not prompt or prompt.strip() == "":
return None, "Please provide a text prompt"
try:
with torch.no_grad():
image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5).images[0]
return image, f"✓ Generated image from prompt: '{prompt}'"
except Exception as e:
return None, f"Error generating image: {str(e)}"
# Combined function: speech -> text -> image
def speech_to_image(audio):
"""Convert speech to text, then generate image from the text"""
# Step 1: Convert speech to text
text_prompt = transcribe_audio(audio)
if text_prompt.startswith("Error"):
return None, text_prompt
# Step 2: Generate image from text
image, status = generate_image_from_text(text_prompt)
return image, f"Transcript: '{text_prompt}'\n\n{status}"
# Gradio interface with tabs
with gr.Blocks(title="AI Image Generation from Speech") as demo:
gr.Markdown("# 🎨 AI Image Generation from Speech")
gr.Markdown("Speak your image description, and the AI will generate an image based on your words!")
with gr.Tab("🎤 Speech to Image"):
gr.Markdown("Record or upload audio with your image description")
audio_input = gr.Audio(label="Record Audio", type="numpy")
generate_btn = gr.Button("Generate Image from Speech", variant="primary")
output_image = gr.Image(label="Generated Image")
output_text = gr.Textbox(label="Status", interactive=False)
generate_btn.click(
fn=speech_to_image,
inputs=audio_input,
outputs=[output_image, output_text]
)
with gr.Tab("⌨️ Text to Image"):
gr.Markdown("Or type a description directly")
text_input = gr.Textbox(
label="Enter Image Description",
placeholder="e.g., a beautiful sunset over mountains",
lines=3
)
text_generate_btn = gr.Button("Generate Image", variant="primary")
text_output_image = gr.Image(label="Generated Image")
text_output_status = gr.Textbox(label="Status", interactive=False)
text_generate_btn.click(
fn=generate_image_from_text,
inputs=text_input,
outputs=[text_output_image, text_output_status]
)
# Launch the interface
if __name__ == "__main__":
demo.launch()
|