app.py

# app.py

import os
import re
import gradio as gr
import torch
from torch import cuda
from math import isclose
import whisper
from PyPDF2 import PdfReader
from PIL import Image
from diffusers import StableDiffusionPipeline
from gtts import gTTS
from moviepy.editor import (
    ImageClip,
    AudioFileClip,
    TextClip,
    CompositeVideoClip,
    concatenate_videoclips
)
from moviepy.video.fx.all import resize

######################################
# 1) SETUP AND MODEL LOADING
######################################

# Check for GPU
device = "cuda" if cuda.is_available() else "cpu"
print(f"Using device: {device}")

# Load Stable Diffusion
pipe = StableDiffusionPipeline.from_pretrained(
    "stabilityai/stable-diffusion-2",
    torch_dtype=torch.float16 if device == "cuda" else torch.float32
)
pipe.to(device)

# (Optional) memory optimizations for low VRAM
pipe.enable_attention_slicing()
pipe.enable_sequential_cpu_offload()

# Load Whisper (not actually used here for transcription, but included if needed)
whisper_model = whisper.load_model("small")

# Make output folders
os.makedirs("images", exist_ok=True)
os.makedirs("videos", exist_ok=True)


######################################
# 2) CORE PDF-TO-VIDEO FUNCTION
######################################

def unify_text_no_newlines(text):
    """Replace any sequence of whitespace/newlines with a single space."""
    return re.sub(r"\s+", " ", text).strip()

def split_into_sentences(text):
    """Split text into sentences by period. Adjust to your needs."""
    parts = re.split(r'\.\s*', text)
    # Clean them up
    sentences = [p.strip() for p in parts if p.strip()]
    return sentences

def repeating_zoom(t, base=1.0, amplitude=0.1, period=4.0):
    """
    Continuously zoom in/out in a triangular wave:
      - base=1.0 => no zoom at the center
      - amplitude=0.1 => up to 1.1, down to 1.0, etc.
      - period=4s => every 4s completes one in/out cycle
    """
    cp = (t % period) / period
    if cp < 0.5:
        # 0..0.5 => scale from base..(base+amplitude)
        up = cp / 0.5  # in [0..1]
        scale = base + amplitude * up
    else:
        # 0.5..1 => scale from (base+amplitude)..base
        down = 1 - ((cp - 0.5) / 0.5)
        scale = base + amplitude * down
    return max(0.01, scale)

def add_subtitles(video_clip, text, duration):
    """Overlay word-by-word subtitles at the bottom."""
    words = text.split()
    if not words:
        return video_clip

    word_duration = duration / len(words)
    subclips = []
    for i, w in enumerate(words):
        start_t = i * word_duration
        txt_clip = (
            TextClip(
                w, fontsize=36, color='white',
                font='Arial', bg_color='black', method='caption'
            )
            .set_start(start_t)
            .set_duration(word_duration)
            .set_position(("center", "bottom"))
        )
        subclips.append(txt_clip)
    final = CompositeVideoClip([video_clip, *subclips])
    return final.set_duration(duration)

def process_pdf_to_video(pdf_file_path):
    """
    1) Extract text from PDF (remove newlines).
    2) Split into sentences.
    3) For each sentence, generate image, TTS, clip.
    4) Concatenate final video.
    5) Return final MP4 path.
    """
    # 1) Extract text
    reader = PdfReader(pdf_file_path)
    raw_text = []
    for page in reader.pages:
        page_text = page.extract_text() or ""
        raw_text.append(page_text)
    text = unify_text_no_newlines(" ".join(raw_text))

    # 2) Split sentences
    sentences = split_into_sentences(text)
    if not sentences:
        raise ValueError("No text found in PDF.")

    # Basic Ghibli prompt
    base_prompt = "Ghibli-style art, soft lighting, whimsical characters, serene environment"
    clips = []

    # 3) Generate data for each sentence
    for idx, sentence in enumerate(sentences):
        if not sentence:
            continue

        # Prompt for Stable Diffusion
        prompt = f"{base_prompt}, {sentence}"
        # Generate image
        image = pipe(
            prompt=prompt,
            num_inference_steps=20
        ).images[0]
        img_path = f"images/clip_{idx+1}.png"
        image.save(img_path)

        # TTS
        audio_path = f"videos/tts_{idx+1}.mp3"
        tts = gTTS(sentence, lang='en')
        tts.save(audio_path)

        # Create Clip
        audio_clip = AudioFileClip(audio_path)
        duration = audio_clip.duration
        if duration < 0.1:
            continue

        img_clip = ImageClip(img_path).set_duration(duration)

        # Apply indefinite zoom in/out
        zoom_clip = img_clip.fx(
            resize,
            lambda t: repeating_zoom(t, base=1.0, amplitude=0.1, period=4.0)
        ).set_audio(audio_clip)

        # Add subtitles
        final_clip = add_subtitles(zoom_clip, sentence, duration)
        clips.append(final_clip)

    # 4) Concatenate all
    if not clips:
        raise ValueError("No valid clips generated.")
    
    combined = concatenate_videoclips(clips, method="compose")
    # Resize to 1280x720
    combined_16_9 = combined.resize((1280, 720))

    # 5) Write out final MP4
    final_path = "videos/final_video.mp4"
    combined_16_9.write_videofile(final_path, fps=24, codec="libx264")
    return final_path


######################################
# 3) GRADIO INTERFACE
######################################

def generate_video_from_pdf(pdf_file):
    """
    This is the function called by Gradio.
    pdf_file is a Gradio 'tempfile' object with .name referencing local path.
    """
    if not pdf_file:
        return "No PDF uploaded."
    try:
        final_video_path = process_pdf_to_video(pdf_file.name)
        return final_video_path  # Gradio can display as a video if we return the path
    except Exception as e:
        return f"Error: {str(e)}"


# Build the Gradio UI
with gr.Blocks() as demo:
    gr.Markdown("# PDF to Ghibli-Style Video")
    pdf_input = gr.File(label="Upload PDF", file_types=[".pdf"])
    generate_btn = gr.Button("Generate Video")
    video_output = gr.Video(label="Output Video")

    # When button is clicked, call generate_video_from_pdf
    generate_btn.click(
        fn=generate_video_from_pdf,
        inputs=pdf_input,
        outputs=video_output
    )

# Launch the Gradio app
def start_app():
    # Note: On Hugging Face Spaces, you typically do 'demo.launch()' 
    # without blocking the main thread.
    demo.launch(server_name="0.0.0.0", server_port=7860)

if __name__ == "__main__":
    start_app()