import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # MUST come before torch / any CUDA-touching import import sys import json import math import tempfile import numpy as np import torch from PIL import Image import gradio as gr # ── Ensure repo root is in sys.path so diffsynth / env / src are importable ── _repo_root = os.path.dirname(os.path.abspath(__file__)) if _repo_root not in sys.path: sys.path.insert(0, _repo_root) from huggingface_hub import snapshot_download # ── Constants ───────────────────────────────────────────────────────────────── WAN_BASE_MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B" ECHO_CKPT_REPO = "Echo-Team/Echo-Memory" ECHO_CKPT_PATH = "context_k1/epoch-0.safetensors" DEFAULT_NEGATIVE_PROMPT = "oversaturated colors, overexposed, static, blurry details" HEIGHT, WIDTH = 352, 640 NUM_FRAMES = 81 FPS = 15 # ── Model loading at module scope (ZeroGPU: .to("cuda") is intercepted) ────── print("[app] Downloading Wan2.1-T2V-1.3B base model...") _base_dir = snapshot_download(WAN_BASE_MODEL_ID) print("[app] Downloading Echo-Memory checkpoint...") _ckpt_dir = snapshot_download(ECHO_CKPT_REPO) _ckpt_path = os.path.join(_ckpt_dir, ECHO_CKPT_PATH) _dit_path = os.path.join(_base_dir, "diffusion_pytorch_model.safetensors") _text_encoder_path = os.path.join(_base_dir, "models_t5_umt5-xxl-enc-bf16.pth") _vae_path = os.path.join(_base_dir, "Wan2.1_VAE.pth") _tokenizer_path = os.path.join(_base_dir, "google", "umt5-xxl") from env.loop_utils import load_pipeline_and_ckpt from env.run_replay_loop_two_chunk import run_one_chunk, encode_context_frames_per_frame from env.memory_baseline_runtime import MemoryProfile, infer_memory_profile_spec from diffsynth import save_video from src.model_training.fov_retrieval import compute_rotation_list # ── Inline helpers from inference/unified_inference.py ────────────────────── def resolve_memory_profile(memory_type: str, ckpt_path: str) -> MemoryProfile: """Resolve memory_type to a MemoryProfile. context_k* use default pipe flags.""" _CONTEXT_K_PROFILES = { "context_k1": MemoryProfile(context_override=1), "context_k5": MemoryProfile(context_override=5), "context_k20": MemoryProfile(context_override=20), } if memory_type in _CONTEXT_K_PROFILES: print(f"[app] Using context learning profile: {memory_type}") return _CONTEXT_K_PROFILES[memory_type] spec = infer_memory_profile_spec(ckpt_path) if spec is not None: print(f"[app] Auto-detected memory profile: {spec.profile_id}") return spec.profile return MemoryProfile() def apply_profile_to_pipe(pipe, profile: MemoryProfile) -> None: """Apply a MemoryProfile directly to the pipeline object.""" pipe.use_framepack_memory = bool(profile.use_framepack_memory) pipe.context_temporal_decay = float(profile.context_temporal_decay or 1.0) pipe.context_attention_weight = float(profile.context_attention_weight or 1.0) pipe.use_framepack_length_compress = bool(profile.use_framepack_length_compress) pipe.framepack_ratio = int(profile.framepack_ratio or 2) pipe.use_spatial_memory = bool(profile.use_spatial_memory) pipe.spatial_memory_tokens = int(profile.spatial_memory_tokens or 64) if profile.spatial_memory_inject_mode: pipe.spatial_memory_inject_mode = str(profile.spatial_memory_inject_mode) pipe.use_spatial_memory_legacy = bool(profile.use_spatial_memory_legacy) pipe.use_block_wise_ssm = bool(getattr(profile, "use_block_wise_ssm", False)) pipe.use_videossm_hybrid = bool(getattr(profile, "use_videossm_hybrid", False)) print("[app] Loading pipeline (DiT -> cuda)...") pipe = load_pipeline_and_ckpt( ckpt_path=_ckpt_path, dit_path=_dit_path, text_encoder_path=_text_encoder_path, vae_path=_vae_path, device="cuda", add_action_attn=False, action_use_temporal_attention=True, tokenizer_path=_tokenizer_path, ) # Apply the memory profile for context_k1 _profile = resolve_memory_profile("context_k1", _ckpt_path) apply_profile_to_pipe(pipe, _profile) print("[app] Model loaded and memory profile applied.") def _build_rotation_action(deg: float, clockwise: bool, num_frames: int = 81) -> dict: """Build a uniform yaw-rotation action dictionary for `num_frames` frames. Args: deg: rotation magnitude in degrees. clockwise: if True, rotate clockwise (negative yaw); else counter-clockwise. num_frames: number of frames in the chunk. Returns: dict mapping frame index (str) -> 12-D RT list. """ denom = max(1, num_frames - 1) actions = {} for i in range(num_frames): yaw = (i / denom) * (-deg if clockwise else deg) actions[str(i)] = compute_rotation_list([0.0, 0.0, 0.0, yaw]) return actions @spaces.GPU(duration=180) def generate( context_image: Image.Image | None, prompt: str, rotation_direction: str, rotation_degrees: float, seed: int, num_inference_steps: int, cfg_scale: float, progress=gr.Progress(track_tqdm=True), ): """Generate an action-conditioned video from an initial frame and a text prompt. Args: context_image: Initial frame (first image of the video). prompt: Text description of the scene. rotation_direction: Camera rotation direction ("Left (CCW)" or "Right (CW)"). rotation_degrees: Total rotation in degrees (e.g. 45). seed: RNG seed for reproducibility. num_inference_steps: Number of diffusion denoising steps. cfg_scale: Classifier-free guidance scale. """ if context_image is None: return None, "Please provide an initial image." if not prompt or not prompt.strip(): return None, "Please provide a text prompt." clockwise = "Right" in rotation_direction or "CW" in rotation_direction deg = float(rotation_degrees) # Resize context image to model resolution ctx_pil = context_image.convert("RGB").resize((WIDTH, HEIGHT), Image.LANCZOS) # Encode context frame through VAE print("[generate] Encoding context image...") pipe.load_models_to_device(["vae"]) with torch.no_grad(): context_latents = encode_context_frames_per_frame(pipe, [ctx_pil], pipe.device) num_context_frames = 1 identity_rt = [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0] context_actions_t = torch.tensor([identity_rt], dtype=torch.float32) # Build camera rotation action JSON and write to temp file cam_pose_actions = _build_rotation_action(deg, clockwise, NUM_FRAMES) action_tmp = tempfile.NamedTemporaryFile(suffix=".json", delete=False, mode="w") json.dump(cam_pose_actions, action_tmp) action_tmp.close() action_path = action_tmp.name # Generate video print(f"[generate] Generating {NUM_FRAMES} frames @ {WIDTH}x{HEIGHT}, rotation={deg}° {'CW' if clockwise else 'CCW'}") frames = run_one_chunk( pipe=pipe, prompt=prompt, use_negative_prompt=DEFAULT_NEGATIVE_PROMPT, action_path=action_path, context_latents=context_latents, num_context_frames=num_context_frames, context_actions_t=context_actions_t, chunk_frames=NUM_FRAMES, h=HEIGHT, w=WIDTH, seed=int(seed), sigma_shift=15.0, num_inference_steps=int(num_inference_steps), cfg_scale=float(cfg_scale), log_prefix="[generate]", ) # Save to temporary file tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) tmp.close() save_video(frames, tmp.name, fps=FPS, quality=5) print(f"[generate] Video saved to {tmp.name}") return tmp.name, f"Generated {len(frames)} frames with {deg}° {'clockwise' if clockwise else 'counter-clockwise'} rotation." # ── Gradio UI ──────────────────────────────────────────────────────────────── CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks() as demo: gr.Markdown("# 🧠 Echo-Memory: Action-Conditioned World Model") gr.Markdown( "Generate a video from an initial frame, a text prompt, and a camera rotation action. " "Based on the [Echo-Memory](https://huggingface.co/papers/2606.09803) paper — " "a controlled study of memory in action world models using the Wan 2.1 1.3B backbone." ) with gr.Row(): with gr.Column(scale=1): context_image = gr.Image(label="Initial Frame", type="pil", height=300) prompt = gr.Textbox( label="Text Prompt", placeholder="A toy bear on a table, the camera rotates around it", lines=2, ) with gr.Row(): rotation_direction = gr.Radio( label="Camera Rotation", choices=["Left (CCW)", "Right (CW)"], value="Left (CCW)", ) rotation_degrees = gr.Slider( label="Rotation Degrees", minimum=5, maximum=90, value=45, step=5, ) run_btn = gr.Button("Generate Video", variant="primary") with gr.Column(scale=1): video_output = gr.Video(label="Generated Video", height=300) status_text = gr.Textbox(label="Status", interactive=False) with gr.Accordion("Advanced Settings", open=False): with gr.Row(): seed = gr.Number(label="Seed", value=42, precision=0) num_inference_steps = gr.Slider( label="Inference Steps", minimum=10, maximum=100, value=50, step=5, ) cfg_scale = gr.Slider( label="CFG Scale", minimum=1.0, maximum=10.0, value=5.0, step=0.5, ) gr.Examples( examples=[ ["examples/1774363417.png", "A toy bear on a table, the camera rotates around it", "Left (CCW)", 45], ["examples/1774363487.png", "A decorative object on a surface, rotating view", "Right (CW)", 45], ["examples/1774363572.png", "A scene with objects on a table, camera pans", "Left (CCW)", 30], ], inputs=[context_image, prompt, rotation_direction, rotation_degrees], ) gr.Markdown( "---\n" "**Model:** [Echo-Team/Echo-Memory](https://huggingface.co/Echo-Team/Echo-Memory) · " "**Backbone:** [Wan-AI/Wan2.1-T2V-1.3B](https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B) · " "**Paper:** [arXiv:2606.09803](https://arxiv.org/abs/2606.09803) · " "**Code:** [GitHub](https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory)" ) run_btn.click( fn=generate, inputs=[context_image, prompt, rotation_direction, rotation_degrees, seed, num_inference_steps, cfg_scale], outputs=[video_output, status_text], api_name="generate", ) demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)