Update app.py
Browse files
app.py
CHANGED
|
@@ -5,38 +5,55 @@ import os
|
|
| 5 |
import cv2
|
| 6 |
import subprocess
|
| 7 |
|
| 8 |
-
# ---
|
| 9 |
try:
|
| 10 |
nlp = spacy.load("en_core_web_sm")
|
| 11 |
except OSError:
|
| 12 |
subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
|
| 13 |
nlp = spacy.load("en_core_web_sm")
|
| 14 |
|
| 15 |
-
# ---
|
| 16 |
ASSET_MAP = {
|
| 17 |
"man": "assets/characters/man.png",
|
| 18 |
"woman": "assets/characters/woman.png",
|
| 19 |
"dog": "assets/characters/dog.png",
|
| 20 |
"park": "assets/backgrounds/park.jpg",
|
| 21 |
-
"office": "assets/backgrounds/office.jpg"
|
|
|
|
|
|
|
|
|
|
| 22 |
}
|
| 23 |
|
| 24 |
FRAME_FOLDER = "frames"
|
| 25 |
VIDEO_OUTPUT = "generated_video.mp4"
|
| 26 |
|
| 27 |
-
# --- Extract characters
|
| 28 |
def extract_entities(prompt):
|
| 29 |
doc = nlp(prompt)
|
| 30 |
characters = []
|
| 31 |
scenes = []
|
|
|
|
|
|
|
| 32 |
for ent in doc.ents:
|
| 33 |
if ent.label_ in ["PERSON", "ORG"]:
|
| 34 |
characters.append(ent.text.lower())
|
| 35 |
elif ent.label_ in ["LOC", "GPE", "FAC"]:
|
| 36 |
scenes.append(ent.text.lower())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
return characters, scenes
|
| 38 |
|
| 39 |
-
# --- Compose a single frame ---
|
| 40 |
def compose_frame(background_path, character_paths, output_path, char_positions=None):
|
| 41 |
bg = Image.open(background_path).convert('RGBA')
|
| 42 |
for idx, char_path in enumerate(character_paths):
|
|
@@ -45,7 +62,7 @@ def compose_frame(background_path, character_paths, output_path, char_positions=
|
|
| 45 |
bg.paste(char_img, pos, char_img)
|
| 46 |
bg.save(output_path)
|
| 47 |
|
| 48 |
-
# --- Create video from frames ---
|
| 49 |
def create_video_from_frames(frame_folder, output_path, fps=24):
|
| 50 |
images = sorted([img for img in os.listdir(frame_folder) if img.endswith(".png")])
|
| 51 |
if not images:
|
|
@@ -62,30 +79,29 @@ def create_video_from_frames(frame_folder, output_path, fps=24):
|
|
| 62 |
# --- Main function triggered by Gradio ---
|
| 63 |
def generate_video(prompt):
|
| 64 |
characters, scenes = extract_entities(prompt)
|
| 65 |
-
|
| 66 |
-
if not scenes:
|
| 67 |
-
return None, "No scene detected! Please include a place in your prompt."
|
| 68 |
|
| 69 |
os.makedirs(FRAME_FOLDER, exist_ok=True)
|
| 70 |
|
| 71 |
bg_path = ASSET_MAP.get(scenes[0], ASSET_MAP["park"])
|
| 72 |
char_paths = [ASSET_MAP.get(char, ASSET_MAP["man"]) for char in characters]
|
| 73 |
|
| 74 |
-
total_frames = 48 #
|
| 75 |
for i in range(total_frames):
|
| 76 |
-
positions = [(100 + i*2, 200) for _ in char_paths]
|
| 77 |
frame_path = os.path.join(FRAME_FOLDER, f"frame_{i:03d}.png")
|
| 78 |
compose_frame(bg_path, char_paths, frame_path, char_positions=positions)
|
| 79 |
|
| 80 |
create_video_from_frames(FRAME_FOLDER, VIDEO_OUTPUT)
|
| 81 |
-
|
|
|
|
|
|
|
| 82 |
|
| 83 |
-
# --- Gradio interface ---
|
| 84 |
iface = gr.Interface(
|
| 85 |
fn=generate_video,
|
| 86 |
inputs=gr.Textbox(lines=3, placeholder="Describe your scene here..."),
|
| 87 |
outputs=[gr.Video(), gr.Textbox()],
|
| 88 |
-
title="Text to Video AI App"
|
| 89 |
)
|
| 90 |
|
| 91 |
if __name__ == "__main__":
|
|
|
|
| 5 |
import cv2
|
| 6 |
import subprocess
|
| 7 |
|
| 8 |
+
# --- Load SpaCy model dynamically (avoids build-time download issues) ---
|
| 9 |
try:
|
| 10 |
nlp = spacy.load("en_core_web_sm")
|
| 11 |
except OSError:
|
| 12 |
subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
|
| 13 |
nlp = spacy.load("en_core_web_sm")
|
| 14 |
|
| 15 |
+
# --- Define asset mapping (character and background files) ---
|
| 16 |
ASSET_MAP = {
|
| 17 |
"man": "assets/characters/man.png",
|
| 18 |
"woman": "assets/characters/woman.png",
|
| 19 |
"dog": "assets/characters/dog.png",
|
| 20 |
"park": "assets/backgrounds/park.jpg",
|
| 21 |
+
"office": "assets/backgrounds/office.jpg",
|
| 22 |
+
"home": "assets/backgrounds/home.jpg",
|
| 23 |
+
"school": "assets/backgrounds/school.jpg",
|
| 24 |
+
"street": "assets/backgrounds/street.jpg"
|
| 25 |
}
|
| 26 |
|
| 27 |
FRAME_FOLDER = "frames"
|
| 28 |
VIDEO_OUTPUT = "generated_video.mp4"
|
| 29 |
|
| 30 |
+
# --- Extract characters and scenes from prompt ---
|
| 31 |
def extract_entities(prompt):
|
| 32 |
doc = nlp(prompt)
|
| 33 |
characters = []
|
| 34 |
scenes = []
|
| 35 |
+
|
| 36 |
+
# Named Entity Recognition
|
| 37 |
for ent in doc.ents:
|
| 38 |
if ent.label_ in ["PERSON", "ORG"]:
|
| 39 |
characters.append(ent.text.lower())
|
| 40 |
elif ent.label_ in ["LOC", "GPE", "FAC"]:
|
| 41 |
scenes.append(ent.text.lower())
|
| 42 |
+
|
| 43 |
+
# If no scenes found → keyword matching from ASSET_MAP keys
|
| 44 |
+
if not scenes:
|
| 45 |
+
for keyword in ASSET_MAP.keys():
|
| 46 |
+
if keyword in prompt.lower() and keyword in ["park", "office", "home", "school", "street"]:
|
| 47 |
+
scenes.append(keyword)
|
| 48 |
+
break
|
| 49 |
+
|
| 50 |
+
# If still no scene → fallback default
|
| 51 |
+
if not scenes:
|
| 52 |
+
scenes.append("park")
|
| 53 |
+
|
| 54 |
return characters, scenes
|
| 55 |
|
| 56 |
+
# --- Compose a single image frame ---
|
| 57 |
def compose_frame(background_path, character_paths, output_path, char_positions=None):
|
| 58 |
bg = Image.open(background_path).convert('RGBA')
|
| 59 |
for idx, char_path in enumerate(character_paths):
|
|
|
|
| 62 |
bg.paste(char_img, pos, char_img)
|
| 63 |
bg.save(output_path)
|
| 64 |
|
| 65 |
+
# --- Create a video from image frames ---
|
| 66 |
def create_video_from_frames(frame_folder, output_path, fps=24):
|
| 67 |
images = sorted([img for img in os.listdir(frame_folder) if img.endswith(".png")])
|
| 68 |
if not images:
|
|
|
|
| 79 |
# --- Main function triggered by Gradio ---
|
| 80 |
def generate_video(prompt):
|
| 81 |
characters, scenes = extract_entities(prompt)
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
os.makedirs(FRAME_FOLDER, exist_ok=True)
|
| 84 |
|
| 85 |
bg_path = ASSET_MAP.get(scenes[0], ASSET_MAP["park"])
|
| 86 |
char_paths = [ASSET_MAP.get(char, ASSET_MAP["man"]) for char in characters]
|
| 87 |
|
| 88 |
+
total_frames = 48 # 2 sec @ 24fps; increase to 2880 for 2 min
|
| 89 |
for i in range(total_frames):
|
| 90 |
+
positions = [(100 + i*2, 200) for _ in char_paths]
|
| 91 |
frame_path = os.path.join(FRAME_FOLDER, f"frame_{i:03d}.png")
|
| 92 |
compose_frame(bg_path, char_paths, frame_path, char_positions=positions)
|
| 93 |
|
| 94 |
create_video_from_frames(FRAME_FOLDER, VIDEO_OUTPUT)
|
| 95 |
+
|
| 96 |
+
details = f"Characters detected: {characters if characters else 'default'}, Scene: {scenes[0]}"
|
| 97 |
+
return VIDEO_OUTPUT, details
|
| 98 |
|
| 99 |
+
# --- Gradio interface setup ---
|
| 100 |
iface = gr.Interface(
|
| 101 |
fn=generate_video,
|
| 102 |
inputs=gr.Textbox(lines=3, placeholder="Describe your scene here..."),
|
| 103 |
outputs=[gr.Video(), gr.Textbox()],
|
| 104 |
+
title="Text to Video AI App (with fallback scenes)"
|
| 105 |
)
|
| 106 |
|
| 107 |
if __name__ == "__main__":
|