#!/usr/bin/env python3 """SCAIL-2 video generation script. Directly calls ComfyUI nodes — no server, no API, no GUI. Uses cuda:0 only. Usage: CUDA_VISIBLE_DEVICES=0 python3 generate.py --pose pose.mp4 --ref ref.png -o output.mp4 """ import argparse import json import os import sys # ── cuda:0 only ── os.environ["CUDA_VISIBLE_DEVICES"] = "0" import cv2 import numpy as np import torch # ── ComfyUI path ── COMFY_DIR = "/home/ubuntu/ComfyUI" sys.path.insert(0, COMFY_DIR) import folder_paths import nodes from comfy_extras.nodes_custom_sampler import KSamplerSelect, BasicScheduler, SamplerCustom from comfy_extras.nodes_scail import WanSCAILToVideo from comfy_extras.nodes_post_processing import ColorTransfer # ────────────────────────── helpers ────────────────────────── def load_video_frames(path, max_frames=None, target_fps=None): """Load video → ComfyUI IMAGE tensor [B, H, W, C] float32 0-1.""" cap = cv2.VideoCapture(path) if not cap.isOpened(): raise FileNotFoundError(f"Cannot open video: {path}") src_fps = cap.get(cv2.CAP_PROP_FPS) frames = [] while True: ret, frame = cap.read() if not ret: break frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frames.append(frame) cap.release() if not frames: raise ValueError(f"No frames in {path}") # fps conversion if target_fps and src_fps > 0 and src_fps != target_fps: step = src_fps / target_fps indices = np.arange(0, len(frames), step).astype(int) indices = indices[indices < len(frames)] frames = [frames[i] for i in indices] if max_frames: frames = frames[:max_frames] tensor = torch.from_numpy(np.stack(frames)).float() / 255.0 # B,H,W,C print(f"Loaded video: {tensor.shape[0]} frames, {tensor.shape[2]}x{tensor.shape[1]}, " f"src_fps={src_fps:.1f}") return tensor def load_image(path): """Load image → ComfyUI IMAGE tensor [1, H, W, C] float32 0-1.""" img = cv2.imread(path, cv2.IMREAD_UNCHANGED) if img is None: raise FileNotFoundError(f"Cannot open image: {path}") img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) if img.ndim == 3 else img tensor = torch.from_numpy(img).float() / 255.0 if tensor.ndim == 2: # grayscale tensor = tensor.unsqueeze(-1) tensor = tensor.unsqueeze(0) # 1,H,W,C print(f"Loaded image: {tensor.shape[2]}x{tensor.shape[1]}") return tensor def load_reference_images(path, target_h, target_w): """Load one image OR a directory of images → batched tensor [N, H, W, C]. SCAIL-2 multi-reference: first image = primary ref, rest = additional views. All images are resized+center-cropped to (target_w, target_h). """ if os.path.isdir(path): files = sorted( os.path.join(path, f) for f in os.listdir(path) if f.lower().endswith((".png", ".jpg", ".jpeg", ".webp")) ) if not files: raise FileNotFoundError(f"No images in directory: {path}") else: files = [path] tensors = [] for f in files: img = cv2.imread(f, cv2.IMREAD_COLOR) if img is None: raise FileNotFoundError(f"Cannot open image: {f}") img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) h, w = img.shape[:2] # center-crop to target aspect ratio, then resize target_ar = target_w / target_h src_ar = w / h if src_ar > target_ar: # too wide → crop sides new_w = int(h * target_ar) x0 = (w - new_w) // 2 img = img[:, x0:x0 + new_w] else: # too tall → crop top/bottom new_h = int(w / target_ar) y0 = (h - new_h) // 2 img = img[y0:y0 + new_h, :] img = cv2.resize(img, (target_w, target_h), interpolation=cv2.INTER_LANCZOS4) t = torch.from_numpy(img).float() / 255.0 tensors.append(t) batch = torch.stack(tensors) # N,H,W,C print(f"Loaded {batch.shape[0]} reference image(s) from {path} " f"(resized to {target_w}x{target_h})") return batch def save_video(tensor, path, fps=24): """Save ComfyUI IMAGE tensor [B, H, W, C] float32 0-1 → mp4.""" arr = (tensor.clone().cpu().numpy() * 255).astype(np.uint8) # B,H,W,C h, w = arr.shape[1], arr.shape[2] fourcc = cv2.VideoWriter_fourcc(*"mp4v") writer = cv2.VideoWriter(path, fourcc, fps, (w, h)) for frame in arr: writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)) writer.release() print(f"Saved: {path} ({arr.shape[0]} frames, {w}x{h}, {fps} fps)") # ────────────────────────── main ────────────────────────── def main(): p = argparse.ArgumentParser(description="SCAIL-2 video generation") p.add_argument("--pose", required=True, help="Pose driving video (mp4)") p.add_argument("--ref", required=True, help="Reference image OR directory of reference images " "(first = primary, rest = additional views)") p.add_argument("-o", "--output", default="output.mp4") p.add_argument("--positive", default="masterpiece, best quality, high quality, detailed") p.add_argument("--negative", default="") p.add_argument("--width", type=int, default=512) p.add_argument("--height", type=int, default=896) p.add_argument("--chunk-length", type=int, default=81) p.add_argument("--overlap", type=int, default=5) p.add_argument("--seed", type=int, default=42) p.add_argument("--cfg", type=float, default=1.0) p.add_argument("--steps", type=int, default=6) p.add_argument("--sampler", default="euler") p.add_argument("--scheduler", default="simple") p.add_argument("--shift", type=float, default=5.0, help="ModelSamplingSD3 shift for flow model (0 to disable)") p.add_argument("--fps", type=int, default=24) p.add_argument("--max-frames", type=int, default=None) p.add_argument("--replacement-mode", action="store_true") p.add_argument("--lora", default=None) p.add_argument("--lora-strength", type=float, default=0.8) p.add_argument("--mask", action="store_true", help="Generate SAM3 pose/reference masks (cached on disk)") p.add_argument("--mask-cache", default=None, help="Mask cache directory (default: ./mask_cache)") p.add_argument("--config", default=None, help="JSON config file (overrides CLI)") args = p.parse_args() if args.config: with open(args.config) as f: cfg = json.load(f) for k, v in cfg.items(): setattr(args, k, v) device = torch.device("cuda:0") print(f"Using device: {device}") # ── 1. Load inputs ── print("\n=== Loading inputs ===") pose_video = load_video_frames(args.pose, max_frames=args.max_frames) ref_image = load_reference_images(args.ref, args.height, args.width) # ── 2. SAM3 masks FIRST (before the 14B model eats the GPU) ── pose_mask = ref_mask = None if args.mask: print("\n=== SAM3 masks ===") import masks as mask_mod kwargs = {} if args.mask_cache: kwargs["cache_dir"] = args.mask_cache pose_mask, ref_mask = mask_mod.get_masks( pose_path=args.pose, ref_path=args.ref, pose_video=pose_video, ref_images=ref_image, width=args.width, height=args.height, replacement_mode=args.replacement_mode, **kwargs) # ── 3. Load models ── print("\n=== Loading models ===") # Diffusion model (SCAIL-2) import importlib.util _kjnodes_spec = importlib.util.spec_from_file_location( 'kjnodes_model_loader', os.path.join(COMFY_DIR, 'custom_nodes/comfyui-kjnodes/nodes/model_optimization_nodes.py') ) _kjnodes_mod = importlib.util.module_from_spec(_kjnodes_spec) _kjnodes_spec.loader.exec_module(_kjnodes_mod) model_loader = _kjnodes_mod.DiffusionModelLoaderKJ() model = model_loader.patch_and_load( model_name="wan2.1_14B_SCAIL_2_fp8_scaled.safetensors", weight_dtype="default", compute_dtype="default", patch_cublaslinear=False, sage_attention="disabled", enable_fp16_accumulation=False, )[0] print(f" Model loaded") # LoRA if args.lora: from nodes import LoraLoaderModelOnly lora_loader = LoraLoaderModelOnly() model = lora_loader.load_lora_model_only( model=model, lora_name=args.lora, strength_model=args.lora_strength, )[0] print(f" LoRA loaded: {args.lora} @ {args.lora_strength}") # ModelSamplingSD3 shift (required for Wan/flow models) if args.shift > 0: from comfy_extras.nodes_model_advanced import ModelSamplingSD3 model = ModelSamplingSD3().patch(model=model, shift=args.shift)[0] print(f" ModelSamplingSD3 shift={args.shift}") # VAE from nodes import VAELoader vae_loader = VAELoader() vae = vae_loader.load_vae("wan_2.1_vae.safetensors")[0] print(f" VAE loaded") # CLIP (text encoder) from nodes import CLIPLoader clip_loader = CLIPLoader() clip = clip_loader.load_clip( clip_name="umt5_xxl_fp8_e4m3fn_scaled.safetensors", type="wan", )[0] print(f" CLIP loaded") # CLIP Vision from nodes import CLIPVisionLoader, CLIPVisionEncode clip_vision_loader = CLIPVisionLoader() clip_vision = clip_vision_loader.load_clip("clip_vision_h_fp16.safetensors")[0] print(f" CLIP Vision loaded") # ── 4. Text encoding ── print("\n=== Encoding prompts ===") from nodes import CLIPTextEncode text_encoder = CLIPTextEncode() positive = text_encoder.encode(clip=clip, text=args.positive)[0] negative = text_encoder.encode(clip=clip, text=args.negative)[0] print(f" Positive: {args.positive[:60]}...") print(f" Negative: {args.negative[:60]}...") # CLIP Vision encode clip_vision_encode = CLIPVisionEncode() clip_vision_output = clip_vision_encode.encode( clip_vision=clip_vision, image=ref_image, crop="center" )[0] print(f" CLIP Vision encoded") # ── 4. Sampler & sigmas ── sampler = KSamplerSelect.execute(sampler_name=args.sampler).args[0] sigmas = BasicScheduler.execute( model=model, scheduler=args.scheduler, steps=args.steps, denoise=1.0 ).args[0] # ── 5. Generate (chunked loop) ── print(f"\n=== Generating ({pose_video.shape[0]} frames, " f"chunk={args.chunk_length}, overlap={args.overlap}) ===") _scail_spec = importlib.util.spec_from_file_location( 'scail_auto_extend', os.path.join(COMFY_DIR, 'custom_nodes/scail-auto-extend/__init__.py') ) _scail_mod = importlib.util.module_from_spec(_scail_spec) _scail_spec.loader.exec_module(_scail_mod) gen = _scail_mod.SCAILAutoExtend() with torch.inference_mode(): output_frames, frame_count = gen.generate( model=model, positive=positive, negative=negative, vae=vae, sampler=sampler, sigmas=sigmas, pose_video=pose_video, width=args.width, height=args.height, noise_seed=args.seed, cfg=args.cfg, chunk_length=args.chunk_length, overlap=args.overlap, seed_mode="increment", color_transfer=True, reference_image=ref_image, reference_image_mask=ref_mask, pose_video_mask=pose_mask, clip_vision_output=clip_vision_output, replacement_mode=args.replacement_mode, ) # ── 6. Save ── print(f"\n=== Saving {frame_count} frames ===") save_video(output_frames, args.output, fps=args.fps) print("Done!") if __name__ == "__main__": main()