Update app.py
Browse files
app.py
CHANGED
|
@@ -1,27 +1,26 @@
|
|
| 1 |
-
import
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
import
|
| 5 |
import torch
|
|
|
|
| 6 |
from min_dalle import MinDalle
|
| 7 |
-
from
|
| 8 |
from PIL import Image, ImageDraw, ImageFont
|
| 9 |
-
import textwrap
|
| 10 |
from mutagen.mp3 import MP3
|
| 11 |
from gtts import gTTS
|
| 12 |
from pydub import AudioSegment
|
| 13 |
-
import
|
| 14 |
-
import
|
| 15 |
-
import nltk
|
| 16 |
-
import subprocess
|
| 17 |
-
import shutil
|
| 18 |
import matplotlib.pyplot as plt
|
| 19 |
-
import gc #
|
|
|
|
| 20 |
from audio import *
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
multiprocessing.set_start_method("spawn")
|
| 24 |
-
|
|
|
|
| 25 |
if os.environ.get("SPACES_ZERO_GPU") is not None:
|
| 26 |
import spaces
|
| 27 |
else:
|
|
@@ -31,45 +30,63 @@ else:
|
|
| 31 |
def wrapper(*args, **kwargs):
|
| 32 |
return func(*args, **kwargs)
|
| 33 |
return wrapper
|
| 34 |
-
# Download necessary NLTK data
|
| 35 |
-
try:
|
| 36 |
-
nltk.data.find('tokenizers/punkt')
|
| 37 |
-
except LookupError:
|
| 38 |
-
nltk.download('punkt')
|
| 39 |
-
|
| 40 |
-
description = "Video Story Generator with Audio \n PS: Generation of video by using Artifical Intellingence by dalle-mini and distilbart and gtss "
|
| 41 |
-
title = "Video Story Generator with Audio by using dalle-mini and distilbart and gtss "
|
| 42 |
|
| 43 |
-
#
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
def log_gpu_memory():
|
|
|
|
| 54 |
if torch.cuda.is_available():
|
| 55 |
print(subprocess.check_output('nvidia-smi').decode('utf-8'))
|
| 56 |
else:
|
| 57 |
print("CUDA is not available. Cannot log GPU memory.")
|
| 58 |
|
| 59 |
-
#
|
| 60 |
-
# Check for GPU availability
|
| 61 |
def check_gpu_availability():
|
|
|
|
| 62 |
if torch.cuda.is_available():
|
| 63 |
print(f"CUDA devices: {torch.cuda.device_count()}")
|
| 64 |
print(f"Current device: {torch.cuda.current_device()}")
|
| 65 |
print(torch.cuda.get_device_properties(torch.cuda.current_device()))
|
| 66 |
else:
|
| 67 |
print("CUDA is not available. Running on CPU.")
|
|
|
|
| 68 |
check_gpu_availability()
|
| 69 |
-
|
|
|
|
| 70 |
def initialize_min_dalle_with_gpu():
|
|
|
|
| 71 |
@spaces.GPU(duration=60 * 3)
|
| 72 |
def load_model():
|
|
|
|
| 73 |
return MinDalle(
|
| 74 |
is_mega=True,
|
| 75 |
models_root='pretrained',
|
|
@@ -80,9 +97,10 @@ def initialize_min_dalle_with_gpu():
|
|
| 80 |
)
|
| 81 |
return load_model()
|
| 82 |
|
| 83 |
-
# Initialize MinDalle
|
| 84 |
min_dalle_model = initialize_min_dalle_with_gpu()
|
| 85 |
|
|
|
|
| 86 |
def generate_image_with_min_dalle(
|
| 87 |
model: MinDalle,
|
| 88 |
text: str,
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import multiprocessing
|
| 3 |
+
import subprocess
|
| 4 |
+
import nltk
|
| 5 |
import torch
|
| 6 |
+
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
| 7 |
from min_dalle import MinDalle
|
| 8 |
+
from moviepy.editor import VideoFileClip
|
| 9 |
from PIL import Image, ImageDraw, ImageFont
|
|
|
|
| 10 |
from mutagen.mp3 import MP3
|
| 11 |
from gtts import gTTS
|
| 12 |
from pydub import AudioSegment
|
| 13 |
+
import textwrap
|
| 14 |
+
import gradio as gr
|
|
|
|
|
|
|
|
|
|
| 15 |
import matplotlib.pyplot as plt
|
| 16 |
+
import gc # Garbage collector
|
| 17 |
+
from huggingface_hub import snapshot_download
|
| 18 |
from audio import *
|
| 19 |
+
|
| 20 |
+
# Ensure proper multiprocessing start method
|
| 21 |
+
multiprocessing.set_start_method("spawn", force=True)
|
| 22 |
+
|
| 23 |
+
# GPU Fallback Setup
|
| 24 |
if os.environ.get("SPACES_ZERO_GPU") is not None:
|
| 25 |
import spaces
|
| 26 |
else:
|
|
|
|
| 30 |
def wrapper(*args, **kwargs):
|
| 31 |
return func(*args, **kwargs)
|
| 32 |
return wrapper
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
+
# Download necessary NLTK data
|
| 35 |
+
def setup_nltk():
|
| 36 |
+
"""Ensure required NLTK data is available."""
|
| 37 |
+
try:
|
| 38 |
+
nltk.data.find('tokenizers/punkt')
|
| 39 |
+
except LookupError:
|
| 40 |
+
nltk.download('punkt')
|
| 41 |
+
|
| 42 |
+
setup_nltk()
|
| 43 |
+
|
| 44 |
+
# Constants
|
| 45 |
+
DESCRIPTION = (
|
| 46 |
+
"Video Story Generator with Audio\n"
|
| 47 |
+
"PS: Generation of video by using Artificial Intelligence via dalle-mini, distilbart, and GTTS."
|
| 48 |
+
)
|
| 49 |
+
TITLE = "Video Story Generator with Audio by using dalle-mini, distilbart, and GTTS."
|
| 50 |
+
|
| 51 |
+
# Load Tokenizer and Model for Text Summarization
|
| 52 |
+
def load_text_summarization_model():
|
| 53 |
+
"""Load the tokenizer and model for text summarization."""
|
| 54 |
+
print("Loading text summarization model...")
|
| 55 |
+
tokenizer = AutoTokenizer.from_pretrained("sshleifer/distilbart-cnn-12-6")
|
| 56 |
+
model = AutoModelForSeq2SeqLM.from_pretrained("sshleifer/distilbart-cnn-12-6")
|
| 57 |
+
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
| 58 |
+
print(f"Using device: {device}")
|
| 59 |
+
model.to(device)
|
| 60 |
+
return tokenizer, model, device
|
| 61 |
+
|
| 62 |
+
tokenizer, model, device = load_text_summarization_model()
|
| 63 |
+
|
| 64 |
+
# Log GPU Memory (optional, for debugging)
|
| 65 |
def log_gpu_memory():
|
| 66 |
+
"""Log GPU memory usage."""
|
| 67 |
if torch.cuda.is_available():
|
| 68 |
print(subprocess.check_output('nvidia-smi').decode('utf-8'))
|
| 69 |
else:
|
| 70 |
print("CUDA is not available. Cannot log GPU memory.")
|
| 71 |
|
| 72 |
+
# Check GPU Availability
|
|
|
|
| 73 |
def check_gpu_availability():
|
| 74 |
+
"""Print GPU availability and device details."""
|
| 75 |
if torch.cuda.is_available():
|
| 76 |
print(f"CUDA devices: {torch.cuda.device_count()}")
|
| 77 |
print(f"Current device: {torch.cuda.current_device()}")
|
| 78 |
print(torch.cuda.get_device_properties(torch.cuda.current_device()))
|
| 79 |
else:
|
| 80 |
print("CUDA is not available. Running on CPU.")
|
| 81 |
+
|
| 82 |
check_gpu_availability()
|
| 83 |
+
|
| 84 |
+
# GPU-Safe MinDalle Model Loading
|
| 85 |
def initialize_min_dalle_with_gpu():
|
| 86 |
+
"""Load the MinDalle model with GPU support."""
|
| 87 |
@spaces.GPU(duration=60 * 3)
|
| 88 |
def load_model():
|
| 89 |
+
print("Loading MinDalle model...")
|
| 90 |
return MinDalle(
|
| 91 |
is_mega=True,
|
| 92 |
models_root='pretrained',
|
|
|
|
| 97 |
)
|
| 98 |
return load_model()
|
| 99 |
|
| 100 |
+
# Initialize MinDalle Model
|
| 101 |
min_dalle_model = initialize_min_dalle_with_gpu()
|
| 102 |
|
| 103 |
+
|
| 104 |
def generate_image_with_min_dalle(
|
| 105 |
model: MinDalle,
|
| 106 |
text: str,
|