Update app.py

app.py

@@ -1 +1,629 @@
-14,21
+import gradio as gr
+import torch
+import numpy as np
+import cv2
+from PIL import Image
+import json
+import os
+from typing import List, Dict, Any
+import tempfile
+import subprocess
+from pathlib import Path
+import spaces
+import gc
+
+# All open-source HuggingFace models
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+from diffusers import (
+    StableDiffusionPipeline, 
+    StableDiffusionXLPipeline,
+    StableVideoDiffusionPipeline,
+    AnimateDiffPipeline,
+    MotionAdapter,
+    DDIMScheduler,
+    DPMSolverMultistepScheduler
+)
+from diffusers.utils import export_to_video
+import soundfile as sf
+
+class CartoonFilmGenerator:
+    def __init__(self):
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.temp_dir = tempfile.mkdtemp()
+        
+        # Model configurations for ZeroGPU optimization
+        self.models_loaded = False
+        
+    @spaces.GPU
+    def load_models(self):
+        """Load models on-demand for ZeroGPU efficiency"""
+        if self.models_loaded:
+            return
+            
+        print("🚀 Loading open-source models...")
+        
+        try:
+            # 1. Image generation - Using SD 1.5 for better compatibility
+            print("📷 Loading image generator...")
+            self.image_generator = StableDiffusionPipeline.from_pretrained(
+                "runwayml/stable-diffusion-v1-5",
+                torch_dtype=torch.float16,
+                use_safetensors=True,
+                safety_checker=None,
+                requires_safety_checker=False
+            ).to(self.device)
+            
+            # Enable memory optimizations
+            self.image_generator.enable_vae_slicing()
+            if hasattr(self.image_generator, 'enable_vae_tiling'):
+                self.image_generator.enable_vae_tiling()
+            
+            print("✅ Image generator loaded successfully")
+            
+        except Exception as e:
+            print(f"❌ Image generator failed: {e}")
+            self.image_generator = None
+        
+        try:
+            # 2. Video generation - Simplified AnimateDiff setup
+            print("🎥 Loading video generator...")
+            
+            # Use a more stable motion adapter
+            adapter = MotionAdapter.from_pretrained(
+                "guoyww/animatediff-motion-adapter-v1-5-2",
+                torch_dtype=torch.float16
+            )
+            
+            self.video_generator = AnimateDiffPipeline.from_pretrained(
+                "runwayml/stable-diffusion-v1-5",
+                motion_adapter=adapter,
+                torch_dtype=torch.float16,
+                safety_checker=None,
+                requires_safety_checker=False
+            ).to(self.device)
+            
+            # Use DPM solver for better stability
+            self.video_generator.scheduler = DPMSolverMultistepScheduler.from_pretrained(
+                "runwayml/stable-diffusion-v1-5",
+                subfolder="scheduler"
+            )
+            
+            # Enable memory optimizations
+            self.video_generator.enable_vae_slicing()
+            if hasattr(self.video_generator, 'enable_vae_tiling'):
+                self.video_generator.enable_vae_tiling()
+            
+            print("✅ Video generator loaded successfully")
+            
+        except Exception as e:
+            print(f"❌ Video generator failed: {e}")
+            self.video_generator = None
+        
+        # 3. Skip TTS for now due to loading issues
+        print("🔇 Skipping TTS model due to loading issues")
+        self.tts_model = None
+        
+        self.models_loaded = True
+        print("🎬 Model loading completed!")
+    
+    def clear_gpu_memory(self):
+        """Clear GPU memory between operations"""
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            gc.collect()
+    
+    def create_structured_script(self, original: str) -> Dict[str, Any]:
+        """Create structured script data from user input"""
+        
+        # Extract key elements from the script
+        words = original.lower().split()
+        
+        # Determine main character and setting
+        if any(word in words for word in ['boy', 'man', 'hero', 'prince', 'knight']):
+            main_char = "brave young hero"
+        elif any(word in words for word in ['girl', 'woman', 'princess', 'heroine']):
+            main_char = "brave young heroine"
+        elif any(word in words for word in ['robot', 'android', 'machine']):
+            main_char = "friendly robot"
+        elif any(word in words for word in ['animal', 'cat', 'dog', 'fox']):
+            main_char = "cute animal character"
+        else:
+            main_char = "friendly protagonist"
+        
+        # Determine setting
+        if any(word in words for word in ['forest', 'woods', 'trees', 'jungle']):
+            setting = "magical forest"
+        elif any(word in words for word in ['city', 'town', 'urban', 'street']):
+            setting = "bustling city"
+        elif any(word in words for word in ['space', 'stars', 'planet', 'galaxy']):
+            setting = "cosmic space adventure"
+        elif any(word in words for word in ['ocean', 'sea', 'underwater', 'beach']):
+            setting = "underwater world"
+        elif any(word in words for word in ['mountain', 'cave', 'valley']):
+            setting = "mountain landscape"
+        else:
+            setting = "colorful fantasy world"
+        
+        # Determine story theme
+        if any(word in words for word in ['treasure', 'gold', 'find', 'search']):
+            theme = "treasure hunting adventure"
+        elif any(word in words for word in ['friend', 'friendship', 'help', 'together']):
+            theme = "friendship and teamwork"
+        elif any(word in words for word in ['magic', 'magical', 'spell', 'wizard']):
+            theme = "magical discovery"
+        elif any(word in words for word in ['save', 'rescue', 'danger', 'protect']):
+            theme = "heroic rescue mission"
+        else:
+            theme = "exciting adventure"
+        
+        # Create 6 scenes for faster processing
+        scenes = []
+        scene_templates = [
+            f"Introduction - Meet our {main_char} in the {setting}",
+            f"Discovery - {main_char} discovers the {theme}",
+            f"Journey begins - Starting the adventure in {setting}",
+            f"Challenge - Facing obstacles during the {theme}",
+            f"Resolution - {main_char} succeeds in the {theme}",
+            f"Celebration - Happy ending in the {setting}"
+        ]
+        
+        for i, template in enumerate(scene_templates):
+            scenes.append({
+                "scene_number": i + 1,
+                "description": template,
+                "characters_present": [main_char] if i % 2 == 0 else [main_char, "supporting friend"],
+                "dialogue": [
+                    {"character": main_char, "text": f"This is an exciting moment in scene {i+1}!"}
+                ],
+                "background": f"{setting} with {['morning', 'day', 'afternoon', 'evening', 'sunset', 'twilight'][i]} lighting",
+                "mood": ["cheerful", "curious", "adventurous", "determined", "triumphant", "joyful"][i],
+                "duration": "25"  # Shorter scenes
+            })
+        
+        return {
+            "title": f"The {theme.title()} of the {main_char.title()}",
+            "characters": [
+                {
+                    "name": main_char,
+                    "description": f"Cartoon-style {main_char} with big expressive eyes, bright colorful outfit, friendly smile, 2D animation style",
+                    "personality": "brave, kind, determined, adventurous"
+                },
+                {
+                    "name": "supporting friend", 
+                    "description": "Helpful cartoon companion with warm bright colors, friendly appearance, loyal sidekick, 2D animation style",
+                    "personality": "loyal, wise, encouraging, funny"
+                }
+            ],
+            "scenes": scenes,
+            "setting": setting,
+            "theme": theme,
+            "style": "Bright and colorful 2D cartoon animation, family-friendly, expressive characters, Disney-Pixar inspired"
+        }
+    
+    @spaces.GPU
+    def generate_character_images(self, characters: List[Dict]) -> Dict[str, str]:
+        """Generate character images"""
+        self.load_models()
+        character_images = {}
+        
+        if not self.image_generator:
+            print("❌ No image generator available")
+            return character_images
+        
+        for character in characters:
+            prompt = f"cartoon character design, {character['description']}, character sheet, multiple poses, white background, 2D animation style, high quality, colorful, Disney style"
+            negative_prompt = "realistic, 3D render, dark, scary, blurry, low quality, inappropriate"
+            
+            try:
+                print(f"🎨 Generating character: {character['name']}")
+                image = self.image_generator(
+                    prompt=prompt,
+                    negative_prompt=negative_prompt,
+                    num_inference_steps=20,
+                    guidance_scale=7.5,
+                    height=512,
+                    width=512
+                ).images[0]
+                
+                char_path = f"{self.temp_dir}/character_{character['name'].replace(' ', '_')}.png"
+                image.save(char_path)
+                character_images[character['name']] = char_path
+                print(f"✅ Generated character: {character['name']}")
+                
+                self.clear_gpu_memory()
+                
+            except Exception as e:
+                print(f"❌ Error generating character {character['name']}: {e}")
+        
+        return character_images
+    
+    @spaces.GPU  
+    def generate_background_images(self, scenes: List[Dict]) -> Dict[int, str]:
+        """Generate background images for each scene"""
+        self.load_models()
+        background_images = {}
+        
+        if not self.image_generator:
+            print("❌ No image generator available")
+            return background_images
+        
+        for scene in scenes:
+            prompt = f"cartoon background scene, {scene['background']}, {scene['mood']} atmosphere, no characters, detailed environment, bright vibrant colors, 2D animation style, Disney background art"
+            negative_prompt = "characters, people, realistic, dark, scary, low quality, blurry"
+            
+            try:
+                print(f"🖼️ Generating background for scene {scene['scene_number']}")
+                image = self.image_generator(
+                    prompt=prompt,
+                    negative_prompt=negative_prompt,
+                    num_inference_steps=18,
+                    guidance_scale=7.0,
+                    height=384,  # 16:10 aspect ratio
+                    width=640
+                ).images[0]
+                
+                bg_path = f"{self.temp_dir}/background_scene_{scene['scene_number']}.png"
+                image.save(bg_path)
+                background_images[scene['scene_number']] = bg_path
+                print(f"✅ Generated background for scene {scene['scene_number']}")
+                
+                self.clear_gpu_memory()
+                
+            except Exception as e:
+                print(f"❌ Error generating background for scene {scene['scene_number']}: {e}")
+        
+        return background_images
+    
+    @spaces.GPU
+    def generate_scene_videos(self, scenes: List[Dict], character_images: Dict, background_images: Dict) -> List[str]:
+        """Generate videos for each scene"""
+        self.load_models()
+        scene_videos = []
+        
+        for scene in scenes:
+            try:
+                if self.video_generator:
+                    print(f"🎬 Generating video for scene {scene['scene_number']}")
+                    
+                    # Create prompt for scene animation
+                    characters_text = ", ".join(scene['characters_present'])
+                    prompt = f"cartoon animation, {characters_text} in {scene['background']}, {scene['mood']} mood, 2D animated style, smooth gentle motion, Disney animation, family friendly"
+                    negative_prompt = "realistic, 3D, static, blurry, low quality, scary, violent"
+                    
+                    # Generate animated video
+                    video_frames = self.video_generator(
+                        prompt=prompt,
+                        negative_prompt=negative_prompt,
+                        num_frames=8,  # Short clips for memory efficiency
+                        guidance_scale=7.5,
+                        num_inference_steps=12,  # Reduced steps for speed
+                        height=384,
+                        width=640
+                    ).frames[0]
+                    
+                    # Save video
+                    video_path = f"{self.temp_dir}/scene_{scene['scene_number']}.mp4"
+                    export_to_video(video_frames, video_path, fps=4)  # Slow FPS for smooth motion
+                    scene_videos.append(video_path)
+                    print(f"✅ Generated video for scene {scene['scene_number']}")
+                    
+                    self.clear_gpu_memory()
+                    
+                else:
+                    # Fallback: create static video with effects
+                    if scene['scene_number'] in background_images:
+                        print(f"📹 Creating static video for scene {scene['scene_number']}")
+                        video_path = self.create_enhanced_static_video(
+                            Image.open(background_images[scene['scene_number']]), 
+                            int(scene.get('duration', 25)), 
+                            scene['scene_number'],
+                            scene['mood']
+                        )
+                        if video_path:
+                            scene_videos.append(video_path)
+                            print(f"✅ Created static video for scene {scene['scene_number']}")
+                
+            except Exception as e:
+                print(f"❌ Error in scene {scene['scene_number']}: {e}")
+                # Create simple fallback video
+                if scene['scene_number'] in background_images:
+                    try:
+                        video_path = self.create_simple_static_video(
+                            Image.open(background_images[scene['scene_number']]), 
+                            int(scene.get('duration', 25)), 
+                            scene['scene_number']
+                        )
+                        if video_path:
+                            scene_videos.append(video_path)
+                            print(f"✅ Created fallback video for scene {scene['scene_number']}")
+                    except Exception as e2:
+                        print(f"��� Fallback video failed: {e2}")
+        
+        return scene_videos
+    
+    def create_enhanced_static_video(self, image: Image.Image, duration: int, scene_num: int, mood: str) -> str:
+        """Create enhanced static video with mood-based effects"""
+        video_path = f"{self.temp_dir}/scene_{scene_num}.mp4"
+        
+        try:
+            # Convert PIL to OpenCV
+            img_array = np.array(image.resize((640, 384)))
+            img_array = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
+            
+            # Create video writer
+            fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+            fps = 12
+            out = cv2.VideoWriter(video_path, fourcc, fps, (640, 384))
+            
+            total_frames = duration * fps
+            
+            for i in range(total_frames):
+                frame = img_array.copy()
+                progress = i / total_frames
+                
+                # Apply mood-based effects
+                if mood in ['cheerful', 'joyful']:
+                    # Gentle zoom in
+                    scale = 1.0 + progress * 0.05
+                elif mood in ['adventurous', 'determined']:
+                    # Slight pan effect
+                    shift_x = int(np.sin(progress * 2 * np.pi) * 10)
+                    M = np.float32([[1, 0, shift_x], [0, 1, 0]])
+                    frame = cv2.warpAffine(frame, M, (640, 384))
+                    scale = 1.0
+                elif mood in ['curious']:
+                    # Zoom out slightly
+                    scale = 1.05 - progress * 0.03
+                else:
+                    # Default gentle zoom
+                    scale = 1.0 + progress * 0.02
+                
+                # Apply scaling
+                if scale != 1.0:
+                    h, w = frame.shape[:2]
+                    center_x, center_y = w // 2, h // 2
+                    M = cv2.getRotationMatrix2D((center_x, center_y), 0, scale)
+                    frame = cv2.warpAffine(frame, M, (w, h))
+                
+                out.write(frame)
+            
+            out.release()
+            return video_path
+            
+        except Exception as e:
+            print(f"❌ Enhanced static video creation failed: {e}")
+            return None
+    
+    def create_simple_static_video(self, image: Image.Image, duration: int, scene_num: int) -> str:
+        """Create simple static video as fallback"""
+        video_path = f"{self.temp_dir}/scene_{scene_num}.mp4"
+        
+        try:
+            img_array = np.array(image.resize((640, 384)))
+            img_array = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
+            
+            fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+            fps = 12
+            out = cv2.VideoWriter(video_path, fourcc, fps, (640, 384))
+            
+            # Simple static video
+            for _ in range(duration * fps):
+                out.write(img_array)
+            
+            out.release()
+            return video_path
+            
+        except Exception as e:
+            print(f"❌ Simple static video creation failed: {e}")
+            return None
+    
+    def merge_videos_with_ffmpeg(self, scene_videos: List[str]) -> str:
+        """Merge videos using ffmpeg"""
+        if not scene_videos:
+            print("❌ No videos to merge")
+            return None
+            
+        final_video_path = f"{self.temp_dir}/final_cartoon_film.mp4"
+        
+        try:
+            print("🎞️ Merging videos...")
+            
+            # Create concat file
+            concat_file = f"{self.temp_dir}/concat_list.txt"
+            with open(concat_file, 'w') as f:
+                for video in scene_videos:
+                    if os.path.exists(video):
+                        f.write(f"file '{os.path.abspath(video)}'\n")
+            
+            # Merge videos
+            cmd = [
+                'ffmpeg', '-f', 'concat', '-safe', '0', '-i', concat_file,
+                '-c:v', 'libx264', '-preset', 'fast', '-crf', '23',
+                '-y', final_video_path
+            ]
+            
+            result = subprocess.run(cmd, capture_output=True, text=True)
+            if result.returncode == 0:
+                print("✅ Video merging successful")
+                return final_video_path
+            else:
+                print(f"❌ FFmpeg error: {result.stderr}")
+                return None
+                
+        except Exception as e:
+            print(f"❌ Video merging failed: {e}")
+            return None
+    
+    @spaces.GPU
+    def generate_cartoon_film(self, script: str) -> tuple:
+        """Main function to generate complete cartoon film"""
+        try:
+            print("🎬 Starting cartoon film generation...")
+            
+            # Step 1: Process script
+            print("📝 Creating story structure...")
+            processed_script = self.create_structured_script(script)
+            
+            # Step 2: Generate characters  
+            print("👥 Creating character designs...")
+            character_images = self.generate_character_images(processed_script['characters'])
+            
+            # Step 3: Generate backgrounds
+            print("🏞️ Creating scene backgrounds...")
+            background_images = self.generate_background_images(processed_script['scenes'])
+            
+            # Step 4: Generate scene videos
+            print("🎥 Creating animated scenes...")
+            scene_videos = self.generate_scene_videos(
+                processed_script['scenes'], 
+                character_images, 
+                background_images
+            )
+            
+            # Step 5: Merge final video
+            print("🎞️ Creating final film...")
+            final_video = self.merge_videos_with_ffmpeg(scene_videos)
+            
+            if final_video and os.path.exists(final_video):
+                print("✅ Cartoon film generation complete!")
+                return final_video, processed_script, "✅ Cartoon film generated successfully!"
+            else:
+                print("⚠️ Partial success - some components may be missing")
+                return None, processed_script, "⚠️ Generation completed with some issues"
+                
+        except Exception as e:
+            print(f"❌ Generation failed: {e}")
+            error_info = {
+                "error": True,
+                "message": str(e),
+                "characters": [],
+                "scenes": [],
+                "style": "Error occurred during generation"
+            }
+            return None, error_info, f"❌ Generation failed: {str(e)}"
+
+# Initialize generator
+generator = CartoonFilmGenerator()
+
+@spaces.GPU
+def create_cartoon_film(script):
+    """Gradio interface function"""
+    if not script.strip():
+        empty_response = {
+            "error": True,
+            "message": "No script provided",
+            "characters": [],
+            "scenes": [],
+            "style": "Please enter a script"
+        }
+        return None, empty_response, "❌ Please enter a script"
+    
+    return generator.generate_cartoon_film(script)
+
+# Gradio Interface
+with gr.Blocks(
+    title="🎬 AI Cartoon Film Generator",
+    theme=gr.themes.Soft(),
+    css="""
+    .gradio-container {
+        max-width: 1200px !important;
+    }
+    """
+) as demo:
+    
+    gr.Markdown("""
+    # 🎬 AI Cartoon Film Generator (Optimized & Fixed)
+    
+    Transform your script into a complete cartoon film using stable open-source models!
+    
+    **🔥 Features:**
+    - **Stable Diffusion 1.5** for reliable image generation
+    - **AnimateDiff** for smooth video animation (when available)
+    - **Enhanced static videos** with mood-based effects as fallback
+    - **6 scenes** for optimal processing time (2.5 minutes total)
+    - **ZeroGPU optimized** with robust error handling
+    
+    **✅ Fixed Issues:**
+    - Removed problematic TTS model
+    - Updated MotionAdapter configuration  
+    - Better memory management
+    - Stable fallback options
+    """)
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            script_input = gr.Textbox(
+                label="📝 Your Story Script",
+                placeholder="Enter your story idea! Examples:\n\n• A brave explorer finds a magical forest\n• A robot learns about friendship in the city\n• Two friends help an alien return home\n• A young artist's drawings come to life",
+                lines=6,
+                max_lines=10
+            )
+                
+            generate_btn = gr.Button(
+                "🎬 Generate Cartoon Film", 
+                variant="primary",
+                size="lg"
+            )
+            
+            gr.Markdown("""
+            **⏱️ Processing Time:** 3-5 minutes  
+            **🎥 Output:** 2.5 minute MP4 film (6 scenes)  
+            **📱 Models:** SD 1.5 + AnimateDiff + Enhanced Effects
+            """)
+        
+        with gr.Column(scale=1):
+            video_output = gr.Video(
+                label="🎬 Generated Cartoon Film",
+                height=400
+            )
+            
+            status_output = gr.Textbox(
+                label="📊 Generation Status",
+                lines=2
+            )
+            
+            script_details = gr.JSON(
+                label="📋 Story Structure",
+                visible=True
+            )
+    
+    # Event handlers
+    generate_btn.click(
+        fn=create_cartoon_film,
+        inputs=[script_input],
+        outputs=[video_output, script_details, status_output],
+        show_progress=True
+    )
+    
+    # Example scripts
+    gr.Examples(
+        examples=[
+            ["A brave young explorer discovers a magical forest where talking animals help find a lost treasure."],
+            ["Two best friends go on a space adventure to help a friendly alien return to its home planet."], 
+            ["A small robot in a big city learns about emotions and friendship from a lonely child."],
+            ["A young artist discovers that their drawings come to life and need help solving magical problems."],
+            ["A curious cat and clever mouse team up to save their neighborhood from a mischievous wizard."],
+            ["A kind-hearted dragon who just wants to make friends learns to overcome fear and prejudice."]
+        ],
+        inputs=[script_input],
+        label="💡 Try these example stories:"
+    )
+    
+    gr.Markdown("""
+    ---
+    **🛠️ Technical Stack:**
+    - **Image Generation:** Stable Diffusion 1.5 (reliable & fast)
+    - **Video Animation:** AnimateDiff with DPM scheduler
+    - **Fallback Effects:** Enhanced static videos with mood-based animation
+    - **Memory Management:** Optimized for ZeroGPU constraints
+    - **Error Handling:** Robust fallbacks at every step
+    
+    **💡 Tips for best results:**
+    - Keep scripts simple and clear
+    - Include character types (hero, robot, animal, etc.)
+    - Mention the setting (forest, city, space, etc.)
+    - Add emotion words (brave, curious, friendly, etc.)
+    """)
+
+if __name__ == "__main__":
+    demo.queue(max_size=3).launch()