Spaces:
Running on Zero
Running on Zero
multimodalart HF Staff
Match author's validated inference defaults: 50 steps, CFG 3.5 (anima_minimal_inference_control_net_vace.py)
516e80d verified | import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| os.environ.setdefault("DIFFSYNTH_DOWNLOAD_SOURCE", "huggingface") | |
| os.environ.setdefault("DIFFSYNTH_SKIP_DOWNLOAD", "True") | |
| os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") | |
| import spaces # MUST be first (after os.environ setup) | |
| import random | |
| import time | |
| from pathlib import Path | |
| from typing import Optional | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| import gradio as gr | |
| from PIL import Image | |
| from huggingface_hub import hf_hub_download, snapshot_download | |
| # βββ model constants βββββββββββββββββββββββββββββββββββββββββββββ | |
| BASE_MODEL_ID = "circlestone-labs/Anima" | |
| DIFFUSION_FILE = "split_files/diffusion_models/anima-base-v1.0.safetensors" | |
| TEXT_ENCODER_FILE = "split_files/text_encoders/qwen_3_06b_base.safetensors" | |
| VAE_FILE = "split_files/vae/qwen_image_vae.safetensors" | |
| QWEN_TOKENIZER_ID = "Qwen/Qwen3-0.6B" | |
| T5_TOKENIZER_ID = "google/t5-v1_1-xxl" | |
| CONTROLNET_MODEL_ID = "TaihoC/Anima-ControlNet-VACE-Depth" | |
| CONTROLNET_FILE = "anima-vace-depth.safetensors" | |
| DEFAULT_POSITIVE_PREFIX = "masterpiece, best quality, score_7, safe, " | |
| DEFAULT_NEGATIVE = "worst quality, low quality, score_1, score_2, score_3, artist name" | |
| # βββ paths βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _select_writable_dir(env_name, candidates): | |
| existing = os.getenv(env_name) | |
| if existing: | |
| candidates = [existing] + [c for c in candidates if c != existing] | |
| for c in candidates: | |
| try: | |
| p = Path(c) | |
| p.mkdir(parents=True, exist_ok=True) | |
| test = p / ".write_test" | |
| test.write_text("ok") | |
| test.unlink(missing_ok=True) | |
| return str(p) | |
| except Exception: | |
| continue | |
| fb = Path("/tmp") / env_name.lower() | |
| fb.mkdir(parents=True, exist_ok=True) | |
| return str(fb) | |
| HF_HOME = _select_writable_dir("HF_HOME", ["/data/.cache/huggingface", "/tmp/.cache/huggingface"]) | |
| LOCAL_MODEL_DIR = _select_writable_dir("ANIMA_LOCAL_MODEL_DIR", ["/data/models/anima-vace", "/tmp/models/anima-vace"]) | |
| os.environ["HF_HOME"] = HF_HOME | |
| os.environ.setdefault("HF_HUB_CACHE", str(Path(HF_HOME) / "hub")) | |
| os.environ.setdefault("DIFFSYNTH_MODEL_BASE_PATH", str(Path(LOCAL_MODEL_DIR) / "diffsynth")) | |
| _HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_HUB_TOKEN") | |
| # βββ DiffSynth imports (after spaces) ββββββββββββββββββββββββββββ | |
| from diffsynth.pipelines.anima_image import AnimaImagePipeline, ModelConfig | |
| from diffsynth.models.anima_dit import Block as AnimaBlock, PatchEmbed | |
| # βββ VACE ControlNet module ββββββββββββββββββββββββββββββββββββββ | |
| # Ported from TaihoC/ComfyUI-Advanced-ControlNet-Anima (feat/anima-vace-controlnet) | |
| # The VACE architecture: spaced-block-duplication + zero-conv, where | |
| # each control block mirrors the Anima DiT Block and its output hint | |
| # is added to the corresponding base DiT block's residual stream. | |
| _HINTS_KEY = "anima_vace_hints" | |
| class VACEControlBlock(nn.Module): | |
| """One block of the VACE control branch: a copy of the Anima DiT Block | |
| plus zero-conv projections. Block 0 additionally fuses the control | |
| latent with the base model's patch-embedded input via before_proj.""" | |
| def __init__(self, block: nn.Module, x_dim: int, block_id: int = 0, use_after_proj: bool = True): | |
| super().__init__() | |
| self.block = block | |
| self.block_id = block_id | |
| self.x_dim = x_dim | |
| self.use_after_proj = use_after_proj | |
| if block_id == 0: | |
| self.before_proj = nn.Linear(x_dim, x_dim) | |
| if use_after_proj: | |
| self.after_proj = nn.Linear(x_dim, x_dim) | |
| def init_weights(self): | |
| if hasattr(self.block, "init_weights"): | |
| self.block.init_weights() | |
| if self.block_id == 0: | |
| nn.init.zeros_(self.before_proj.weight) | |
| nn.init.zeros_(self.before_proj.bias) | |
| if self.use_after_proj: | |
| nn.init.zeros_(self.after_proj.weight) | |
| nn.init.zeros_(self.after_proj.bias) | |
| def forward(self, c, x_base, **block_kwargs): | |
| all_c = [] | |
| if self.block_id == 0: | |
| c = self.before_proj(c) + x_base | |
| elif self.use_after_proj: | |
| all_c = list(torch.unbind(c)) | |
| c = all_c.pop(-1) | |
| c = self.block(c, **block_kwargs) | |
| if self.use_after_proj: | |
| c_skip = self.after_proj(c) | |
| all_c = all_c + [c_skip, c] | |
| c = torch.stack(all_c) | |
| else: | |
| c_skip = c | |
| return c_skip, c | |
| class VACEControlNetAnima(nn.Module): | |
| """Standalone VACE control branch: PatchEmbed for the control latent | |
| plus a ModuleList of VACEControlBlock. forward() returns per-base-block | |
| hint tensors keyed by base block index.""" | |
| def __init__( | |
| self, | |
| model_channels=2048, | |
| num_heads=16, | |
| num_blocks=28, | |
| crossattn_emb_channels=1024, | |
| patch_spatial=2, | |
| patch_temporal=1, | |
| latent_channels=16, | |
| vace_block_every_n=7, | |
| condition_strategy="spaced", | |
| use_after_proj=True, | |
| use_adaln_lora=True, | |
| adaln_lora_dim=256, | |
| concat_padding_mask=True, | |
| ): | |
| super().__init__() | |
| self.model_channels = model_channels | |
| self.num_blocks = num_blocks | |
| self.latent_channels = latent_channels | |
| self.vace_block_every_n = vace_block_every_n | |
| self.condition_strategy = condition_strategy | |
| self.concat_padding_mask = concat_padding_mask | |
| if condition_strategy == "spaced": | |
| self.control_layers = list(range(0, num_blocks, vace_block_every_n)) | |
| self.control_layers_mapping = {i: n for n, i in enumerate(self.control_layers)} | |
| else: | |
| n_ctrl = max(1, num_blocks // vace_block_every_n) | |
| self.control_layers = list(range(0, n_ctrl)) | |
| self.control_layers_mapping = {i: i for i in range(n_ctrl)} | |
| assert 0 in self.control_layers, "first control block must map to base block 0" | |
| embed_in_channels = latent_channels + (1 if concat_padding_mask else 0) | |
| self.control_embedder = PatchEmbed( | |
| spatial_patch_size=patch_spatial, | |
| temporal_patch_size=patch_temporal, | |
| in_channels=embed_in_channels, | |
| out_channels=model_channels, | |
| operations=torch.nn, | |
| ) | |
| self.control_blocks = nn.ModuleList([ | |
| VACEControlBlock( | |
| block=AnimaBlock( | |
| x_dim=model_channels, | |
| context_dim=crossattn_emb_channels, | |
| num_heads=num_heads, | |
| mlp_ratio=4.0, | |
| use_adaln_lora=use_adaln_lora, | |
| adaln_lora_dim=adaln_lora_dim, | |
| operations=torch.nn, | |
| ), | |
| x_dim=model_channels, | |
| block_id=i, | |
| use_after_proj=use_after_proj, | |
| ) | |
| for i in range(len(self.control_layers)) | |
| ]) | |
| def forward(self, control_latent, x_base_tokens, emb_B_T_D, crossattn_emb, | |
| rope_emb=None, adaln_lora=None, extra_per_block_pos_emb=None): | |
| cdtype = control_latent.dtype | |
| x_base_tokens = x_base_tokens.to(cdtype) | |
| emb_B_T_D = emb_B_T_D.to(cdtype) | |
| crossattn_emb = crossattn_emb.to(cdtype) | |
| if adaln_lora is not None: | |
| adaln_lora = adaln_lora.to(cdtype) | |
| if extra_per_block_pos_emb is not None: | |
| extra_per_block_pos_emb = extra_per_block_pos_emb.to(cdtype) | |
| if self.concat_padding_mask and control_latent.shape[1] == self.latent_channels: | |
| zeros_mask = torch.zeros( | |
| control_latent.shape[0], 1, *control_latent.shape[2:], | |
| device=control_latent.device, dtype=control_latent.dtype, | |
| ) | |
| control_latent = torch.cat([control_latent, zeros_mask], dim=1) | |
| c = self.control_embedder(control_latent) | |
| rope_for_blocks = rope_emb | |
| if rope_emb is not None and rope_emb.ndim == 4: | |
| rope_for_blocks = rope_emb.unsqueeze(1).unsqueeze(0) | |
| block_kwargs = dict( | |
| emb_B_T_D=emb_B_T_D, | |
| crossattn_emb=crossattn_emb, | |
| rope_emb_L_1_1_D=rope_for_blocks, | |
| adaln_lora_B_T_3D=adaln_lora, | |
| extra_per_block_pos_emb=extra_per_block_pos_emb, | |
| ) | |
| hints = {} | |
| for i, cb in enumerate(self.control_blocks): | |
| c_skip, c = cb(c, x_base_tokens, **block_kwargs) | |
| hints[self.control_layers[i]] = c_skip | |
| return hints | |
| # βββ asset download ββββββββββββββββββββββββββββββββββββββββββββββ | |
| class AssetPaths: | |
| diffusion: str | |
| text_encoder: str | |
| vae: str | |
| qwen_tokenizer_dir: str | |
| t5_tokenizer_dir: str | |
| controlnet_ckpt: str | |
| _TOKENIZER_ALLOW = [ | |
| "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", | |
| "added_tokens.json", "vocab.json", "merges.txt", "config.json", | |
| "*.model", "*.tiktoken", "*.jinja", | |
| ] | |
| def _repo_local_dir(repo_id, suffix=""): | |
| safe = repo_id.replace("/", "--") | |
| if suffix: | |
| safe = f"{safe}--{suffix}" | |
| p = Path(LOCAL_MODEL_DIR) / safe | |
| p.mkdir(parents=True, exist_ok=True) | |
| return p | |
| def _download_file(repo_id, filename, local_dir): | |
| return hf_hub_download(repo_id=repo_id, filename=filename, local_dir=str(local_dir), token=_HF_TOKEN) | |
| def _has_tokenizer_files(path): | |
| if not path.is_dir(): | |
| return False | |
| markers = ("tokenizer.json", "tokenizer.model", "spiece.model", "vocab.json", "merges.txt") | |
| return any((path / m).exists() for m in markers) | |
| def _download_tokenizer(repo_id, subfolder="", suffix="tokenizer"): | |
| cleaned = subfolder.strip("/") | |
| local_dir = _repo_local_dir(repo_id, suffix if not cleaned else f"{suffix}--{cleaned}") | |
| if cleaned: | |
| allow = [f"{cleaned}/{p}" for p in _TOKENIZER_ALLOW] | |
| else: | |
| allow = _TOKENIZER_ALLOW | |
| snapshot_dir = Path(snapshot_download( | |
| repo_id=repo_id, allow_patterns=allow, local_dir=str(local_dir), token=_HF_TOKEN | |
| )) | |
| candidates = [] | |
| if cleaned: | |
| candidates.append(snapshot_dir / cleaned) | |
| candidates.append(snapshot_dir) | |
| for c in candidates: | |
| if _has_tokenizer_files(c): | |
| return str(c) | |
| raise RuntimeError(f"No usable tokenizer files found in {repo_id}") | |
| def _download_assets(): | |
| print(f"[startup] Downloading model assets to {LOCAL_MODEL_DIR}", flush=True) | |
| start = time.time() | |
| anima_dir = _repo_local_dir(BASE_MODEL_ID) | |
| diffusion = _download_file(BASE_MODEL_ID, DIFFUSION_FILE, anima_dir) | |
| text_encoder = _download_file(BASE_MODEL_ID, TEXT_ENCODER_FILE, anima_dir) | |
| vae = _download_file(BASE_MODEL_ID, VAE_FILE, anima_dir) | |
| qwen_dir = _download_tokenizer(QWEN_TOKENIZER_ID, suffix="qwen-tokenizer") | |
| t5_dir = _download_tokenizer(T5_TOKENIZER_ID, suffix="t5-tokenizer") | |
| controlnet_ckpt = _download_file(CONTROLNET_MODEL_ID, CONTROLNET_FILE, _repo_local_dir(CONTROLNET_MODEL_ID)) | |
| elapsed = time.time() - start | |
| print(f"[startup] Downloads ready in {elapsed:.1f}s", flush=True) | |
| return AssetPaths( | |
| diffusion=diffusion, text_encoder=text_encoder, vae=vae, | |
| qwen_tokenizer_dir=qwen_dir, t5_tokenizer_dir=t5_dir, | |
| controlnet_ckpt=controlnet_ckpt, | |
| ) | |
| # βββ pipeline state ββββββββββββββββββββββββββββββββββββββββββββββ | |
| _PIPE: Optional[AnimaImagePipeline] = None | |
| _VACE: Optional[VACEControlNetAnima] = None | |
| _ASSETS: Optional[AssetPaths] = None | |
| _DEPTH_PROCESSOR = None | |
| def _ensure_assets(): | |
| global _ASSETS | |
| if _ASSETS is None: | |
| _ASSETS = _download_assets() | |
| return _ASSETS | |
| def _load_vace(ckpt_path): | |
| """Load the VACE ControlNet from a safetensors checkpoint.""" | |
| from safetensors.torch import safe_open, load_file | |
| meta = {} | |
| try: | |
| with safe_open(ckpt_path, framework="pt") as f: | |
| meta = dict(f.metadata() or {}) | |
| except Exception: | |
| pass | |
| model_channels = int(meta.get("vace.model_channels", "2048")) | |
| num_blocks = int(meta.get("vace.num_blocks", "28")) | |
| latent_channels = int(meta.get("vace.in_channels", "16")) | |
| vace_block_every_n = int(meta.get("vace.block_every_n", "7")) | |
| use_after_proj = meta.get("vace.use_after_proj", "true").lower() == "true" | |
| sd = load_file(ckpt_path) | |
| max_ctrl_idx = -1 | |
| for k in sd: | |
| if k.startswith("control_blocks."): | |
| max_ctrl_idx = max(max_ctrl_idx, int(k.split(".")[1])) | |
| if max_ctrl_idx >= 0: | |
| n_ctrl = max_ctrl_idx + 1 | |
| vace_block_every_n = num_blocks // n_ctrl if n_ctrl > 0 else vace_block_every_n | |
| model = VACEControlNetAnima( | |
| model_channels=model_channels, | |
| num_heads=16, | |
| num_blocks=num_blocks, | |
| crossattn_emb_channels=1024, | |
| patch_spatial=2, | |
| patch_temporal=1, | |
| latent_channels=latent_channels, | |
| vace_block_every_n=vace_block_every_n, | |
| condition_strategy="spaced", | |
| use_after_proj=use_after_proj, | |
| use_adaln_lora=True, | |
| adaln_lora_dim=256, | |
| concat_padding_mask=True, | |
| ) | |
| info = model.load_state_dict(sd, strict=False) | |
| if info.missing_keys: | |
| print(f"[vace] missing keys ({len(info.missing_keys)}): {info.missing_keys[:5]}...", flush=True) | |
| if info.unexpected_keys: | |
| print(f"[vace] unexpected keys ({len(info.unexpected_keys)}): {info.unexpected_keys[:5]}...", flush=True) | |
| model = model.to(torch.bfloat16).to("cuda") | |
| model.eval() | |
| return model | |
| def _install_vace_hooks(pipe, vace): | |
| """Register forward hooks on base DiT blocks to inject VACE hints.""" | |
| dit = pipe.dit | |
| for base_idx in vace.control_layers_mapping: | |
| block = dit.blocks[base_idx] | |
| def make_hook(bi): | |
| def hook(module, args, kwargs, output): | |
| to = kwargs.get("transformer_options") or {} | |
| hints = to.get(_HINTS_KEY) | |
| if not hints: | |
| return output | |
| hint = hints.get(bi) | |
| if hint is None: | |
| return output | |
| return output + hint.to(dtype=output.dtype) | |
| return hook | |
| block.register_forward_hook(make_hook(base_idx), with_kwargs=True) | |
| def _load_pipe(): | |
| global _PIPE, _VACE | |
| if _PIPE is not None: | |
| return _PIPE, _VACE | |
| assets = _ensure_assets() | |
| torch.set_float32_matmul_precision("high") | |
| print("[startup] Loading Anima pipeline", flush=True) | |
| _PIPE = AnimaImagePipeline.from_pretrained( | |
| torch_dtype=torch.bfloat16, | |
| device="cuda", | |
| model_configs=[ | |
| ModelConfig(path=assets.diffusion), | |
| ModelConfig(path=assets.text_encoder), | |
| ModelConfig(path=assets.vae), | |
| ], | |
| tokenizer_config=ModelConfig(path=assets.qwen_tokenizer_dir), | |
| tokenizer_t5xxl_config=ModelConfig(path=assets.t5_tokenizer_dir), | |
| ) | |
| print("[startup] Anima pipeline loaded", flush=True) | |
| print("[startup] Loading VACE ControlNet", flush=True) | |
| _VACE = _load_vace(assets.controlnet_ckpt) | |
| _install_vace_hooks(_PIPE, _VACE) | |
| print("[startup] VACE ControlNet loaded and hooked", flush=True) | |
| return _PIPE, _VACE | |
| # βββ depth preprocessor ββββββββββββββββββββββββββββββββββββββββββ | |
| def _load_depth_processor(): | |
| global _DEPTH_PROCESSOR | |
| if _DEPTH_PROCESSOR is not None: | |
| return _DEPTH_PROCESSOR | |
| from transformers import pipeline as hf_pipeline | |
| # The model card recommends DepthAnything V2; use the Large variant on GPU | |
| # for crisp depth edges (Small produces soft, low-contrast maps that blur | |
| # the control signal). Loaded lazily inside the @spaces.GPU context. | |
| _DEPTH_PROCESSOR = hf_pipeline( | |
| task="depth-estimation", | |
| model="depth-anything/Depth-Anything-V2-Large-hf", | |
| device="cuda", | |
| torch_dtype=torch.float16, | |
| ) | |
| print("[startup] Depth processor loaded", flush=True) | |
| return _DEPTH_PROCESSOR | |
| def _extract_depth(image): | |
| """Extract a depth map from an input image using Depth Anything V2.""" | |
| processor = _load_depth_processor() | |
| if image.mode != "RGB": | |
| image = image.convert("RGB") | |
| result = processor(image) | |
| return result["depth"] | |
| # βββ VACE control injection βββββββββββββββββββββββββββββββββββββ | |
| def _run_vace_control(pipe, vace, control_latent, dit, x_noisy, timestep, context, strength): | |
| """Run the VACE control branch and return per-block hints dict.""" | |
| with torch.no_grad(): | |
| x_5d = x_noisy.unsqueeze(2) if x_noisy.ndim == 4 else x_noisy | |
| x_B_T_H_W_D, rope_emb, extra_pos_emb = dit.prepare_embedded_sequence(x_5d) | |
| t = timestep / 1000 | |
| if t.ndim == 1: | |
| t = t.unsqueeze(1) | |
| # Match the DiT forward: convert timesteps to x dtype, then compute embedding | |
| timesteps_emb = dit.t_embedder[0](t.to(x_B_T_H_W_D.dtype)) | |
| # Timesteps may output float32; cast to match TimestepEmbedding weights | |
| timesteps_emb = timesteps_emb.to(x_B_T_H_W_D.dtype) | |
| t_emb, adaln_lora = dit.t_embedder[1](timesteps_emb) | |
| t_emb = dit.t_embedding_norm(t_emb) | |
| cl = control_latent | |
| if cl.ndim == 4: | |
| cl = cl.unsqueeze(2) | |
| if cl.shape[1] != vace.latent_channels: | |
| pipe.load_models_to_device(['vae']) | |
| cl = pipe.vae.encode(cl.movedim(1, -1)) | |
| if cl.ndim == 4: | |
| cl = cl.unsqueeze(2) | |
| hints = vace( | |
| control_latent=cl.to(device=x_B_T_H_W_D.device, dtype=torch.bfloat16), | |
| x_base_tokens=x_B_T_H_W_D, | |
| emb_B_T_D=t_emb, | |
| crossattn_emb=context, | |
| rope_emb=rope_emb, | |
| adaln_lora=adaln_lora, | |
| extra_per_block_pos_emb=extra_pos_emb, | |
| ) | |
| return {ci: hint * strength for ci, hint in hints.items()} | |
| # βββ Gradio inference βββββββββββββββββββββββββββββββββββββββββββ | |
| def generate( | |
| input_image, | |
| prompt: str, | |
| negative_prompt: str, | |
| controlnet_strength: float, | |
| width: int, | |
| height: int, | |
| steps: int, | |
| cfg_scale: float, | |
| seed: int, | |
| sigma_shift: float = 5.0, | |
| progress: gr.Progress = gr.Progress(track_tqdm=False), | |
| ): | |
| """Generate an anime-style image conditioned on the depth map of an input image. | |
| Args: | |
| input_image: Image to extract depth from for conditioning. | |
| prompt: Text prompt describing the desired output. | |
| negative_prompt: What to avoid in the output. | |
| controlnet_strength: Strength of depth conditioning (0-2). | |
| width: Output image width (512-1536, rounded to 16). | |
| height: Output image height (512-1536, rounded to 16). | |
| steps: Number of denoising steps. | |
| cfg_scale: Classifier-free guidance scale. | |
| seed: Random seed (-1 for random). | |
| sigma_shift: Flow-matching timestep shift (the ControlNet was trained | |
| and evaluated at 5.0 per the model card). | |
| """ | |
| if input_image is None: | |
| raise gr.Error("Please provide an input image for depth extraction.") | |
| if not prompt or not prompt.strip(): | |
| prompt = "1girl, solo, long silver hair, blue eyes, blue dress, underwater, floating hair, refraction, portrait" | |
| prompt = DEFAULT_POSITIVE_PREFIX + prompt.strip() | |
| negative_prompt = (negative_prompt or DEFAULT_NEGATIVE).strip() | |
| # Output follows the INPUT image's aspect ratio; the width/height sliders | |
| # set the pixel budget (target area). This keeps the depth map undistorted. | |
| import math as _math | |
| target_area = float(width) * float(height) | |
| aspect = input_image.width / input_image.height | |
| width = int(round(_math.sqrt(target_area * aspect) / 16) * 16) | |
| height = int(round(_math.sqrt(target_area / aspect) / 16) * 16) | |
| width = max(512, min(1536, width)) | |
| height = max(512, min(1536, height)) | |
| steps = int(max(10, min(60, steps))) | |
| cfg_scale = float(max(1.0, min(8.0, cfg_scale))) | |
| sigma_shift = float(max(1.0, min(8.0, sigma_shift))) | |
| if seed < 0: | |
| seed = random.randint(0, 2**31 - 1) | |
| seed = int(seed) | |
| pipe, vace = _load_pipe() | |
| # Extract depth map from input image | |
| progress(0.1, desc="Extracting depth map") | |
| depth_map = _extract_depth(input_image) | |
| depth_map = depth_map.resize((width, height), Image.LANCZOS) | |
| # Ensure 3-channel RGB for VAE encoding | |
| if depth_map.mode != "RGB": | |
| depth_map = depth_map.convert("RGB") | |
| # VAE-encode the depth map into latent space | |
| progress(0.2, desc="Encoding depth latent") | |
| depth_tensor = pipe.preprocess_image(depth_map).to(device=pipe.device, dtype=pipe.torch_dtype) | |
| with torch.no_grad(): | |
| pipe.load_models_to_device(['vae']) | |
| control_latent = pipe.vae.encode(depth_tensor.unsqueeze(2), device=pipe.device).squeeze(2) | |
| # Monkey-patch model_fn to inject VACE control | |
| original_model_fn = pipe.model_fn | |
| _vace_state = {"control_latent": control_latent, "strength": controlnet_strength, "vace": vace} | |
| def patched_model_fn(dit=None, latents=None, timestep=None, prompt_emb=None, | |
| t5xxl_ids=None, use_gradient_checkpointing=False, | |
| use_gradient_checkpointing_offload=False, **kwargs): | |
| # The base AnimaDiT runs the raw Qwen hidden states through its | |
| # LLMAdapter (preprocess_text_embeds, fused with the T5-XXL token | |
| # embeddings) before the blocks ever see them as crossattn_emb. | |
| # ComfyUI pre-applies this in model_base.Anima.extra_conds, so the | |
| # reference VACE branch receives the ADAPTED context β the control | |
| # blocks were trained against it. Adapt once here and feed the same | |
| # tensor to both the control branch and the base DiT (t5xxl_ids=None | |
| # skips re-adaptation inside AnimaDiT.forward). | |
| context = prompt_emb | |
| if t5xxl_ids is not None: | |
| context = dit.preprocess_text_embeds(prompt_emb, t5xxl_ids) | |
| hints = _run_vace_control( | |
| pipe, _vace_state["vace"], _vace_state["control_latent"], | |
| dit, latents, timestep, context, _vace_state["strength"], | |
| ) | |
| transformer_options = kwargs.get("transformer_options", {}) | |
| transformer_options[_HINTS_KEY] = hints | |
| kwargs["transformer_options"] = transformer_options | |
| latents_5d = latents.unsqueeze(2) | |
| timestep_scaled = timestep / 1000 | |
| model_output = dit( | |
| x=latents_5d, | |
| timesteps=timestep_scaled, | |
| context=context, | |
| t5xxl_ids=None, | |
| use_gradient_checkpointing=use_gradient_checkpointing, | |
| use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, | |
| **kwargs, | |
| ) | |
| return model_output.squeeze(2) | |
| pipe.model_fn = patched_model_fn | |
| try: | |
| progress(0.3, desc="Generating") | |
| with torch.inference_mode(): | |
| image = pipe( | |
| prompt=prompt, | |
| negative_prompt=negative_prompt, | |
| cfg_scale=cfg_scale, | |
| height=height, | |
| width=width, | |
| seed=seed, | |
| num_inference_steps=steps, | |
| sigma_shift=sigma_shift, | |
| ) | |
| finally: | |
| pipe.model_fn = original_model_fn | |
| info = ( | |
| f"**Seed:** `{seed}` \n" | |
| f"**Size:** `{width}Γ{height}` \n" | |
| f"**Steps / CFG / shift:** `{steps}` / `{cfg_scale}` / `{sigma_shift}` \n" | |
| f"**ControlNet strength:** `{controlnet_strength}`" | |
| ) | |
| return image, depth_map, info | |
| # βββ startup βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # On ZeroGPU, CUDA is only available inside @spaces.GPU decorated functions, | |
| # so we only download assets at startup; the pipeline loads lazily on first call. | |
| try: | |
| _ASSETS = _download_assets() | |
| print("[startup] Assets downloaded", flush=True) | |
| except Exception as e: | |
| print(f"[startup] Download failed: {e}", flush=True) | |
| _ASSETS = None | |
| # βββ Gradio UI βββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="Anima ControlNet VACE Depth") as demo: | |
| gr.Markdown( | |
| "# Anima ControlNet VACE Depth\n" | |
| "Generate anime-style images conditioned on the depth structure of an input image. " | |
| "Uses the [Anima](https://huggingface.co/circlestone-labs/Anima) base model with the " | |
| "[VACE Depth ControlNet](https://huggingface.co/TaihoC/Anima-ControlNet-VACE-Depth) adapter.\n\n" | |
| "Upload an image, write a prompt, and the model will generate an anime-style image " | |
| "that follows the depth structure of your input." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| input_image = gr.Image(label="Input image (for depth extraction)", type="pil") | |
| prompt = gr.Textbox( | |
| label="Prompt", | |
| lines=3, | |
| value="1girl, solo, long silver hair, blue eyes, blue dress, underwater, floating hair, refraction, portrait, looking at viewer", | |
| ) | |
| negative_prompt = gr.Textbox( | |
| label="Negative prompt", | |
| lines=2, | |
| value=DEFAULT_NEGATIVE, | |
| ) | |
| controlnet_strength = gr.Slider(0.0, 2.0, value=1.0, step=0.05, | |
| label="ControlNet strength") | |
| generate_btn = gr.Button("Generate", variant="primary") | |
| with gr.Accordion("Advanced settings", open=False): | |
| gr.Markdown( | |
| "*The output matches the input image's aspect ratio; " | |
| "width Γ height set the pixel budget.*" | |
| ) | |
| with gr.Row(): | |
| width = gr.Slider(512, 1536, value=1024, step=16, label="Width (budget)") | |
| height = gr.Slider(512, 1536, value=1024, step=16, label="Height (budget)") | |
| with gr.Row(): | |
| steps = gr.Slider(10, 60, value=50, step=1, label="Steps") | |
| cfg_scale = gr.Slider(1.0, 8.0, value=3.5, step=0.1, label="CFG scale") | |
| with gr.Row(): | |
| sigma_shift = gr.Slider(1.0, 8.0, value=5.0, step=0.5, label="Flow shift", | |
| info="ControlNet trained at 5.0 (model card)") | |
| seed = gr.Number(value=-1, precision=0, label="Seed (-1 random)") | |
| with gr.Column(scale=1): | |
| output_image = gr.Image(label="Generated image", type="pil") | |
| depth_image = gr.Image(label="Depth map (extracted)", type="pil") | |
| info = gr.Markdown() | |
| inputs_list = [input_image, prompt, negative_prompt, controlnet_strength, | |
| width, height, steps, cfg_scale, seed, sigma_shift] | |
| generate_btn.click(generate, inputs=inputs_list, outputs=[output_image, depth_image, info], | |
| show_progress=True) | |
| gr.Examples( | |
| examples=[ | |
| ["examples/depth_landscape.jpg", | |
| "year 2025, newest, highres, safe, 1girl, standing in a misty forest, sunlight through trees, fantasy landscape, detailed background", | |
| DEFAULT_NEGATIVE, 1.0, 1024, 1024, 50, 3.5, 42, 5.0], | |
| ["examples/depth_anime_1.jpg", | |
| "year 2025, newest, masterpiece, best quality, score_7, safe, 1girl, silver hair, red eyes, gothic dress, standing pose, detailed background", | |
| DEFAULT_NEGATIVE, 1.0, 1024, 1024, 50, 3.5, 12345, 5.0], | |
| ["examples/depth_astronaut.jpg", | |
| "year 2025, newest, highres, safe, 1boy, space suit, helmet, standing on alien planet, stars, nebula, sci-fi landscape, dramatic lighting", | |
| DEFAULT_NEGATIVE, 0.8, 1024, 1024, 50, 3.5, -1, 5.0], | |
| ], | |
| inputs=inputs_list, | |
| outputs=[output_image, depth_image, info], | |
| fn=generate, | |
| cache_examples=False, | |
| run_on_click=True, | |
| ) | |
| if __name__ == "__main__": | |
| demo.queue(max_size=20).launch( | |
| ssr_mode=False, | |
| theme=gr.themes.Citrus(), | |
| css=CSS, | |
| ) |