Create sketch_video_app_from_blank_direction.py

sketch_video_app_from_blank_direction.py

@@ -0,0 +1,339 @@
+'''
+conda create --name animeins python=3.10
+conda activate animeins
+pip install ipykernel
+python -m ipykernel install --user --name animeins --display-name "animeins"
+pip install -r requirements.txt
+
+pip install torch==2.1.1 torchvision
+pip install mmcv==2.1.0 -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html
+pip install mmdet
+pip install "numpy<2.0.0"
+pip install moviepy==1.0.3
+pip install "httpx[socks]"
+'''
+
+import gradio as gr
+import os
+import cv2
+import numpy as np
+from PIL import Image
+from typing import Literal
+import pathlib
+from animeinsseg import AnimeInsSeg, AnimeInstances
+from animeinsseg.anime_instances import get_color
+
+# Install required packages
+os.system("mim install mmengine")
+os.system('mim install mmcv==2.1.0')
+os.system("mim install mmdet==3.2.0")
+
+# Download model if not exists
+if not os.path.exists("models"):
+    os.mkdir("models")
+    os.system("huggingface-cli lfs-enable-largefiles .")
+    os.system("git clone https://huggingface.co/dreMaz/AnimeInstanceSegmentation models/AnimeInstanceSegmentation")
+
+# Initialize segmentation model
+ckpt = r'models/AnimeInstanceSegmentation/rtmdetl_e60.ckpt'
+mask_thres = 0.3
+instance_thres = 0.3
+refine_kwargs = {'refine_method': 'refinenet_isnet'}
+net = AnimeInsSeg(ckpt, mask_thr=mask_thres, refine_kwargs=refine_kwargs)
+
+def image_to_sketch(image: np.ndarray) -> np.ndarray:
+    """Convert image to pencil sketch"""
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    inverted = 255 - gray
+    blurred = cv2.GaussianBlur(inverted, (21, 21), 0)
+    inverted_blurred = 255 - blurred
+    sketch = cv2.divide(gray, inverted_blurred, scale=256.0)
+    return cv2.cvtColor(sketch, cv2.COLOR_GRAY2BGR)
+
+def generate_combined_transition_video(
+    original_image: np.ndarray,
+    depth_map: np.ndarray,
+    first_transition: str = 'character_first',
+    second_transition: str = 'character_first',
+    first_direction: str = 'left_to_right',
+    second_direction: str = 'left_to_right',
+    duration_sec: float = 6.0,
+    frame_rate: int = 30,
+    depth_blur: int = 15,
+    debug_visualize: bool = False
+) -> str:
+    """
+    Generate combined transition video with customizable scanline directions
+    """
+    # Convert images to proper format
+    original = cv2.cvtColor(original_image, cv2.COLOR_RGB2BGR)
+    depth_map = cv2.cvtColor(depth_map, cv2.COLOR_RGB2GRAY)
+    
+    # Get sketch version
+    sketch = image_to_sketch(original)
+    h, w = original.shape[:2]
+    
+    # Perform instance segmentation
+    instances: AnimeInstances = net.infer(
+        original,
+        output_type='numpy',
+        pred_score_thr=instance_thres
+    )
+    
+    # Prepare depth map
+    depth_map = cv2.resize(depth_map, (w, h))
+    depth_map = cv2.GaussianBlur(depth_map, (depth_blur, depth_blur), 0)
+    depth_map = depth_map.astype(np.float32) / 255.0
+    
+    def create_scanline_mask(mask, progress, direction):
+        """Create scanline mask based on direction"""
+        scanline_mask = np.zeros_like(mask)
+        
+        if direction == 'left_to_right':
+            scan_width = int(w * progress)
+            scanline_mask[:, :scan_width] = mask[:, :scan_width]
+        elif direction == 'right_to_left':
+            scan_width = int(w * progress)
+            scanline_mask[:, -scan_width:] = mask[:, -scan_width:]
+        elif direction == 'top_to_bottom':
+            scan_height = int(h * progress)
+            scanline_mask[:scan_height, :] = mask[:scan_height, :]
+        elif direction == 'bottom_to_top':
+            scan_height = int(h * progress)
+            scanline_mask[-scan_height:, :] = mask[-scan_height:, :]
+        
+        return scanline_mask
+    
+    def create_layer_masks(transition_type):
+        layer_masks = []
+        layer_depths = []
+        
+        # Process segmented instances
+        if instances.bboxes is not None:
+            for mask in instances.masks:
+                instance_depth = np.mean(depth_map[mask.astype(bool)])
+                
+                if transition_type == 'character_first':
+                    layer_masks.append(mask.astype(np.float32))
+                    layer_depths.append(0)
+                else:
+                    if transition_type == 'near_to_far':
+                        instance_depth = 1.0 - instance_depth
+                    layer_masks.append(mask.astype(np.float32))
+                    layer_depths.append(instance_depth)
+        
+        # Create a full mask for the remaining areas
+        if layer_masks:
+            full_mask = 1.0 - np.clip(np.sum(layer_masks, axis=0), 0, 1)
+        else:
+            full_mask = np.ones((h, w), dtype=np.float32)
+        
+        # Process remaining areas
+        if transition_type == 'character_first':
+            if np.sum(full_mask) > 0:
+                layer_masks.append(full_mask)
+                layer_depths.append(1)
+        else:
+            remaining_depth = depth_map * full_mask
+            num_depth_bands = 10
+            
+            min_depth = np.min(remaining_depth[full_mask > 0]) if np.sum(full_mask) > 0 else 0
+            max_depth = np.max(remaining_depth[full_mask > 0]) if np.sum(full_mask) > 0 else 1
+            depth_bands = np.linspace(min_depth, max_depth, num_depth_bands + 1)
+            
+            for i in range(num_depth_bands):
+                lower = depth_bands[i]
+                upper = depth_bands[i+1]
+                band_mask = ((remaining_depth >= lower) & (remaining_depth < upper)).astype(np.float32)
+                
+                if np.sum(band_mask) > 0:
+                    band_depth = np.mean(remaining_depth[band_mask.astype(bool)])
+                    if transition_type == 'near_to_far':
+                        band_depth = 1.0 - band_depth
+                    
+                    layer_masks.append(band_mask)
+                    layer_depths.append(band_depth)
+        
+        # Sort layers if needed
+        if transition_type != 'character_first' and layer_masks:
+            sorted_indices = np.argsort(layer_depths)
+            layer_masks = [layer_masks[i] for i in sorted_indices]
+        
+        return layer_masks
+    
+    # Get masks for both transitions
+    first_masks = create_layer_masks(first_transition)
+    second_masks = create_layer_masks(second_transition)
+    
+    # Generate video
+    output_path = "output_video.mp4"
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    video = cv2.VideoWriter(output_path, fourcc, frame_rate, (w, h))
+    total_frames = int(duration_sec * frame_rate)
+    half_duration = duration_sec / 2
+    
+    for frame_idx in range(total_frames):
+        current_time = frame_idx / frame_rate
+        
+        # Determine which transition we're in
+        if current_time < half_duration:
+            # First transition: blank to sketch
+            progress = np.clip(current_time / half_duration, 0, 1)
+            num_layers = len(first_masks)
+            layer_duration = half_duration / num_layers if num_layers > 0 else half_duration
+            
+            # Start with blank (white) image
+            blended = np.ones_like(original) * 255
+            
+            for layer_idx, layer_mask in enumerate(first_masks):
+                layer_start = layer_idx * layer_duration
+                layer_progress = np.clip((current_time - layer_start) / layer_duration, 0, 1)
+                
+                # Apply scanline effect based on direction
+                scanline_mask = create_scanline_mask(layer_mask, layer_progress, first_direction)
+                scanline_alpha = np.repeat(scanline_mask[..., np.newaxis], 3, axis=2)
+                
+                blended = blended * (1 - scanline_alpha) + sketch.astype(np.float32) * scanline_alpha
+        else:
+            # Second transition: sketch to original
+            progress = np.clip((current_time - half_duration) / half_duration, 0, 1)
+            num_layers = len(second_masks)
+            layer_duration = half_duration / num_layers if num_layers > 0 else half_duration
+            
+            # Start with sketch
+            blended = sketch.copy().astype(np.float32)
+            
+            for layer_idx, layer_mask in enumerate(second_masks):
+                layer_start = half_duration + layer_idx * layer_duration
+                layer_progress = np.clip((current_time - layer_start) / layer_duration, 0, 1)
+                
+                # Apply scanline effect based on direction
+                scanline_mask = create_scanline_mask(layer_mask, layer_progress, second_direction)
+                scanline_alpha = np.repeat(scanline_mask[..., np.newaxis], 3, axis=2)
+                
+                blended = blended * (1 - scanline_alpha) + original.astype(np.float32) * scanline_alpha
+        
+        blended = np.clip(blended, 0, 255).astype(np.uint8)
+        
+        if debug_visualize:
+            cv2.imshow('Blended', blended)
+            if cv2.waitKey(1) == 27:
+                break
+        
+        video.write(blended)
+    
+    video.release()
+    if debug_visualize:
+        cv2.destroyAllWindows()
+    
+    return output_path
+
+def process_images(
+    original_image,
+    depth_map,
+    first_transition,
+    second_transition,
+    first_direction,
+    second_direction,
+    duration
+):
+    # Convert PIL Images to numpy arrays
+    original_np = np.array(original_image)
+    depth_np = np.array(depth_map)
+    
+    # Generate video
+    video_path = generate_combined_transition_video(
+        original_image=original_np,
+        depth_map=depth_np,
+        first_transition=first_transition,
+        second_transition=second_transition,
+        first_direction=first_direction,
+        second_direction=second_direction,
+        duration_sec=float(duration),
+        debug_visualize=False
+    )
+    
+    return video_path
+
+# Create Gradio interface
+with gr.Blocks() as demo:
+    gr.Markdown("# Anime Image Transition Video Generator")
+    gr.Markdown("Upload an image and its depth map to generate a two-phase transition video:")
+    gr.Markdown("1. From blank to sketch")
+    gr.Markdown("2. From sketch to original image")
+    
+    with gr.Row():
+        with gr.Column():
+            original_image = gr.Image(label="Original Image", type="pil")
+            depth_map = gr.Image(label="Depth Map", type="pil")
+            
+            with gr.Group():
+                gr.Markdown("### First Transition (Blank → Sketch)")
+                first_transition = gr.Radio(
+                    choices=["character_first", "near_to_far", "far_to_near"],
+                    value="character_first",
+                    label="Render Order",
+                    info="How elements appear from blank to sketch"
+                )
+                first_direction = gr.Radio(
+                    choices=["left_to_right", "right_to_left", "top_to_bottom", "bottom_to_top"],
+                    value="left_to_right",
+                    label="Scanline Direction",
+                    info="Direction of the reveal effect"
+                )
+            
+            with gr.Group():
+                gr.Markdown("### Second Transition (Sketch → Original)")
+                second_transition = gr.Radio(
+                    choices=["character_first", "near_to_far", "far_to_near"],
+                    value="character_first",
+                    label="Render Order",
+                    info="How elements transition from sketch to original"
+                )
+                second_direction = gr.Radio(
+                    choices=["left_to_right", "right_to_left", "top_to_bottom", "bottom_to_top"],
+                    value="left_to_right",
+                    label="Scanline Direction",
+                    info="Direction of the reveal effect"
+                )
+            
+            duration = gr.Slider(2, 20, value=6, step=0.5, label="Total Duration (seconds)")
+            submit_btn = gr.Button("Generate Video")
+        
+        with gr.Column():
+            output_video = gr.Video(label="Output Video")
+       
+    submit_btn.click(
+        fn=process_images,
+        inputs=[
+            original_image, 
+            depth_map, 
+            first_transition, 
+            second_transition,
+            first_direction,
+            second_direction,
+            duration
+        ],
+        outputs=output_video
+    )
+
+    # Add examples
+    gr.Examples(
+        [
+            ["化物语封面.jpeg", "化物语封面深度.png", "character_first", "far_to_near", "left_to_right", "right_to_left"],
+            ["可莉风景.png", "可莉风景_depth.png", "near_to_far", "character_first", "top_to_bottom", "bottom_to_top"],
+            ["竹林万叶.jpg", "竹林万叶_depth.png", "far_to_near", "near_to_far", "right_to_left", "left_to_right"],
+            ["重云行秋.jpg", "重云行秋_depth.png", "character_first", "character_first", "bottom_to_top", "top_to_bottom"],
+        ],
+        inputs=[
+            original_image, 
+            depth_map, 
+            first_transition, 
+            second_transition,
+            first_direction,
+            second_direction
+        ]
+    )
+
+if __name__ == "__main__":
+    demo.launch(share=True)