Duplicate from fffiloni/speech-to-image

Co-authored-by: Sylvain Filoni <fffiloni@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,33 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Speech To Image • Community pipeline
+emoji: 🎙🖼🧩
+colorFrom: yellow
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.6
+app_file: app.py
+pinned: false
+duplicated_from: fffiloni/speech-to-image
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,60 @@
+import gradio as gr
+import torch
+import whisper
+from diffusers import DiffusionPipeline
+from transformers import (
+    WhisperForConditionalGeneration,
+    WhisperProcessor,
+)
+
+import os
+MY_SECRET_TOKEN=os.environ.get('HF_TOKEN_SD')
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small").to(device)
+processor = WhisperProcessor.from_pretrained("openai/whisper-small")
+
+diffuser_pipeline = DiffusionPipeline.from_pretrained(
+    "CompVis/stable-diffusion-v1-4",
+    custom_pipeline="speech_to_image_diffusion",
+    speech_model=model,
+    speech_processor=processor,
+    use_auth_token=MY_SECRET_TOKEN,
+    revision="fp16",
+    torch_dtype=torch.float16,
+)
+
+diffuser_pipeline.enable_attention_slicing()
+diffuser_pipeline = diffuser_pipeline.to(device)
+
+
+#————————————————————————————————————————————
+# GRADIO SETUP
+title = "Speech to Diffusion • Community Pipeline"
+description = """
+<p style='text-align: center;'>This demo can generate an image from an audio sample using pre-trained OpenAI whisper-small and Stable Diffusion.<br />
+Community examples consist of both inference and training examples that have been added by the community.<br />
+<a href='https://github.com/huggingface/diffusers/tree/main/examples/community#speech-to-image' target='_blank'> Click here for more information about community pipelines </a>
+</p>
+"""
+article = """
+<p style='text-align: center;'>Community pipeline by Mikail Duzenli • Gradio demo by Sylvain Filoni & Ahsen Khaliq<p>
+"""
+audio_input = gr.Audio(source="microphone", type="filepath")
+image_output = gr.Image()
+
+def speech_to_text(audio_sample):
+  
+  process_audio = whisper.load_audio(audio_sample)
+  output = diffuser_pipeline(process_audio)
+ 
+  print(f"""
+  ————————
+  output: {output}
+  ————————
+  """)
+  
+  return output.images[0]
+
+demo = gr.Interface(fn=speech_to_text, inputs=audio_input, outputs=image_output, title=title, description=description, article=article)
+demo.launch()

requirements.txt

@@ -0,0 +1,7 @@
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch
+scipy
+ftfy
+git+https://github.com/huggingface/transformers
+git+https://github.com/huggingface/diffusers
+git+https://github.com/openai/whisper.git