| import gradio as gr |
| import torch |
| from diffusers import StableDiffusionPipeline |
| from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech |
| import openai |
| import cv2 |
| import numpy as np |
|
|
| |
| sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") |
| if torch.cuda.is_available(): |
| sd_pipe.to("cuda") |
|
|
| |
| processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts") |
| tts_model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts") |
|
|
| |
| openai.api_key = "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" |
|
|
| def generate_image(prompt, emotion): |
| emotion_map = {"Happy": "smiling", "Angry": "frowning", "Sad": "crying"} |
| prompt = f"{prompt}, {emotion_map.get(emotion, '')}" |
| image = sd_pipe(prompt).images[0] |
| return image |
|
|
| def generate_voiceover(text): |
| inputs = processor(text, return_tensors="pt") |
| speech = tts_model.generate(**inputs) |
| return speech |
|
|
| def apply_physics(image, effect): |
| img_array = np.array(image) |
| if effect == "Wind": |
| kernel = np.array([[1, 1, 1], [0, 0, 0], [-1, -1, -1]]) |
| img_array = cv2.filter2D(img_array, -1, kernel) |
| elif effect == "Falling": |
| img_array = np.roll(img_array, 10, axis=0) |
| return img_array |
|
|
| def track_facial_expressions(): |
| cap = cv2.VideoCapture(0) |
| face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml') |
| while True: |
| ret, frame = cap.read() |
| gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) |
| faces = face_cascade.detectMultiScale(gray, 1.3, 5) |
| for (x, y, w, h) in faces: |
| cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2) |
| cv2.imshow('Face Tracking', frame) |
| if cv2.waitKey(1) & 0xFF == ord('q'): |
| break |
| cap.release() |
| cv2.destroyAllWindows() |
|
|
| def animate_scene(prompt, emotion, physics, text): |
| image = generate_image(prompt, emotion) |
| image = apply_physics(image, physics) |
| speech = generate_voiceover(text) |
| return image, speech |
|
|
| iface = gr.Interface( |
| fn=animate_scene, |
| inputs=[ |
| gr.Textbox(label="Character Description"), |
| gr.Radio(["Happy", "Angry", "Sad"], label="Emotion"), |
| gr.Radio(["None", "Wind", "Falling"], label="Physics Effect"), |
| gr.Textbox(label="Dialogue") |
| ], |
| outputs=[ |
| gr.Image(label="Generated Scene"), |
| gr.Audio(label="Voiceover") |
| ] |
| ) |
|
|
| iface.launch() |
|
|
