diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -1,4 +1,6 @@ -# --- PART 1: IMPORTS & SYSTEM SETUP --- +#################################################################################################### +### PART 1: Imports & Basic Utilities +#################################################################################################### import sys from pathlib import Path import uuid @@ -12,72 +14,95 @@ from typing import Any import time from contextlib import contextmanager from gradio.helpers import create_examples -import spaces -import flash_attn_interface -import gradio as gr -import numpy as np -import random -from typing import Optional -from huggingface_hub import hf_hub_download, snapshot_download -from PIL import Image -import imageio -import cv2 -import json - -# افزودن پکیجهای لوکال به مسیر سیستم -current_dir = Path(__file__).parent -sys.path.insert(0, str(current_dir / "packages" / "ltx-pipelines" / "src")) -sys.path.insert(0, str(current_dir / "packages" / "ltx-core" / "src")) @contextmanager def timer(name: str): start = time.time() print(f"{name}...") yield - print(f" -> {name} تکمیل شد در {time.time() - start:.2f} ثانیه") - -def sh(cmd): subprocess.check_call(cmd, shell=True) - + print(f" -> {name} completed in {time.time() - start:.2f} sec") -# --- PART 2: AUDIO HELPER FUNCTIONS --- def _coerce_audio_path(audio_path: Any) -> str: + # Common Gradio case: tuple where first item is the filepath if isinstance(audio_path, tuple) and len(audio_path) > 0: audio_path = audio_path[0] + + # Some gradio versions pass a dict-like object if isinstance(audio_path, dict): + # common keys: "name", "path" audio_path = audio_path.get("name") or audio_path.get("path") + + # pathlib.Path etc. if not isinstance(audio_path, (str, bytes, os.PathLike)): raise TypeError(f"audio_path must be a path-like, got {type(audio_path)}: {audio_path}") + return os.fspath(audio_path) + +#################################################################################################### +### PART 2: Audio Extraction & Matching Utilities +#################################################################################################### def extract_audio_wav_ffmpeg(video_path: str, target_sr: int = 48000) -> str | None: + """ + Extract audio from a video into a temp WAV (mono, target_sr). + Returns path, or None if the video has no audio stream. + """ out_path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name + + # Check if there's an audio stream probe_cmd = [ - "ffprobe", "-v", "error", "-select_streams", "a:0", - "-show_entries", "stream=codec_type", "-of", "default=nw=1:nk=1", video_path, + "ffprobe", "-v", "error", + "-select_streams", "a:0", + "-show_entries", "stream=codec_type", + "-of", "default=nw=1:nk=1", + video_path, ] try: out = subprocess.check_output(probe_cmd).decode("utf-8").strip() - if not out: return None + if not out: + return None except subprocess.CalledProcessError: return None + # Extract + resample + mono cmd = [ - "ffmpeg", "-y", "-v", "error", "-i", video_path, "-vn", "-ac", "1", - "-ar", str(int(target_sr)), "-c:a", "pcm_s16le", out_path + "ffmpeg", "-y", "-v", "error", + "-i", video_path, + "-vn", + "-ac", "1", + "-ar", str(int(target_sr)), + "-c:a", "pcm_s16le", + out_path ] subprocess.check_call(cmd) return out_path -def match_audio_to_duration(audio_path: str, target_seconds: float, target_sr: int = 48000, - to_mono: bool = True, pad_mode: str = "silence", device: str = "cuda"): +def match_audio_to_duration( + audio_path: str, + target_seconds: float, + target_sr: int = 48000, + to_mono: bool = True, + pad_mode: str = "silence", # "silence" | "repeat" + device: str = "cuda", +): + """ + Load audio, resample, (optionally) mono, then trim/pad to exactly target_seconds. + Returns: (waveform[T] or [1,T], sr) + """ audio_path = _coerce_audio_path(audio_path) - wav, sr = torchaudio.load(audio_path) + + wav, sr = torchaudio.load(audio_path) # [C, T] float32 CPU + + # Resample to target_sr (recommended so duration math is stable) if sr != target_sr: wav = torchaudio.functional.resample(wav, sr, target_sr) sr = target_sr + + # Mono (common expectation; if your model supports stereo, set to_mono=False) if to_mono and wav.shape[0] > 1: - wav = wav.mean(dim=0, keepdim=True) - + wav = wav.mean(dim=0, keepdim=True) # [1, T] + + # Exact target length in samples target_len = int(round(target_seconds * sr)) cur_len = wav.shape[-1] @@ -86,101 +111,106 @@ def match_audio_to_duration(audio_path: str, target_seconds: float, target_sr: i elif cur_len < target_len: pad_len = target_len - cur_len if pad_mode == "repeat" and cur_len > 0: + # Repeat then cut to exact length reps = (target_len + cur_len - 1) // cur_len wav = wav.repeat(1, reps)[..., :target_len] else: + # Silence pad wav = F.pad(wav, (0, pad_len)) - return wav.to(device, non_blocking=True), sr - - -# --- PART 3: VIDEO HELPER FUNCTIONS (FFMPEG) --- -def probe_video_duration_seconds(video_path: str) -> float: - cmd = [ - "ffprobe", "-v", "error", "-select_streams", "v:0", - "-show_entries", "format=duration", "-of", "json", video_path, - ] - out = subprocess.check_output(cmd).decode("utf-8") - return float(json.loads(out)["format"]["duration"]) - -def trim_video_to_seconds_ffmpeg(video_path: str, target_seconds: float, fps: float = None) -> str: - target_seconds = max(0.01, float(target_seconds)) - out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name - vf = [] - if fps is not None: vf.append(f"fps={float(fps)}") - vf_str = ",".join(vf) if vf else None - cmd = ["ffmpeg", "-y", "-v", "error", "-i", video_path, "-t", f"{target_seconds:.6f}"] - if vf_str: cmd += ["-vf", vf_str] - cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-preset", "veryfast", "-crf", "18", "-an", out_path] - subprocess.check_call(cmd) - return out_path -def extract_first_frame_png(video_path: str) -> str: - out_path = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name - cmd = ["ffmpeg", "-y", "-v", "error", "-i", video_path, "-frames:v", "1", out_path] - subprocess.check_call(cmd) - return out_path + # move to device + wav = wav.to(device, non_blocking=True) + return wav, sr -def load_video_frames(video_path: str): - frames = [] - with imageio.get_reader(video_path) as reader: - for frame in reader: - frames.append(frame) - return frames +def sh(cmd): subprocess.check_call(cmd, shell=True) -def write_video_mp4(frames_float_01, fps: float, out_path: str): - frames_uint8 = [(f * 255).astype(np.uint8) for f in frames_float_01] - with imageio.get_writer(out_path, fps=fps, macro_block_size=1) as writer: - for fr in frames_uint8: - writer.append_data(fr) - return out_path +#################################################################################################### +### PART 3: LTX Pipeline Imports & Constants +#################################################################################################### +# Add packages to Python path +current_dir = Path(__file__).parent +sys.path.insert(0, str(current_dir / "packages" / "ltx-pipelines" / "src")) +sys.path.insert(0, str(current_dir / "packages" / "ltx-core" / "src")) -# --- PART 4: LTX PIPELINE IMPORTS & CONSTANTS --- +import spaces +import flash_attn_interface +import time +import gradio as gr +import numpy as np +import random +import torch +from typing import Optional +from pathlib import Path +import torchaudio +from huggingface_hub import hf_hub_download, snapshot_download from ltx_pipelines.distilled import DistilledPipeline from ltx_core.model.video_vae import TilingConfig from ltx_core.model.audio_vae.ops import AudioProcessor from ltx_core.loader.primitives import LoraPathStrengthAndSDOps from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP from ltx_pipelines.utils.constants import ( - DEFAULT_SEED, DEFAULT_1_STAGE_HEIGHT, DEFAULT_1_STAGE_WIDTH, - DEFAULT_NUM_FRAMES, DEFAULT_FRAME_RATE, DEFAULT_LORA_STRENGTH, + DEFAULT_SEED, + DEFAULT_1_STAGE_HEIGHT, + DEFAULT_1_STAGE_WIDTH , + DEFAULT_NUM_FRAMES, + DEFAULT_FRAME_RATE, + DEFAULT_LORA_STRENGTH, ) from ltx_core.loader.single_gpu_model_builder import enable_only_lora -from ltx_core.model.audio_vae import decode_audio, encode_audio +from ltx_core.model.audio_vae import decode_audio +from ltx_core.model.audio_vae import encode_audio +from PIL import Image + +MAX_SEED = np.iinfo(np.int32).max +# Import from public LTX-2 package +# Install with: pip install git+https://github.com/Lightricks/LTX-2.git from ltx_pipelines.utils import ModelLedger from ltx_pipelines.utils.helpers import generate_enhanced_prompt -from controlnet_aux import CannyDetector, MidasDetector -from dwpose import DwposeDetector +import imageio +import cv2 -MAX_SEED = np.iinfo(np.int32).max +# HuggingFace Hub defaults DEFAULT_REPO_ID = "Lightricks/LTX-2" DEFAULT_GEMMA_REPO_ID = "unsloth/gemma-3-12b-it-qat-bnb-4bit" DEFAULT_CHECKPOINT_FILENAME = "ltx-2-19b-dev.safetensors" -# --- PART 5: MODEL DOWNLOAD & CACHE LOGIC --- +#################################################################################################### +### PART 4: Download Helpers +#################################################################################################### def get_hub_or_local_checkpoint(repo_id: str, filename: str): - print(f"در حال دانلود {filename} از {repo_id}...") + """Download from HuggingFace Hub.""" + print(f"Downloading {filename} from {repo_id}...") ckpt_path = hf_hub_download(repo_id=repo_id, filename=filename) - print(f"دانلود شد در: {ckpt_path}") + print(f"Downloaded to {ckpt_path}") return ckpt_path def download_gemma_model(repo_id: str): - print(f"در حال دانلود مدل Gemma از {repo_id}...") + """Download the full Gemma model directory.""" + print(f"Downloading Gemma model from {repo_id}...") local_dir = snapshot_download(repo_id=repo_id) - print(f"مدل Gemma دانلود شد در: {local_dir}") + print(f"Gemma model downloaded to {local_dir}") return local_dir -# --- PART 6: GLOBAL MODEL LOADING (TEXT ENCODER) --- +#################################################################################################### +### PART 5: Text Encoder Initialization +#################################################################################################### +# Initialize model ledger and text encoder at startup (load once, keep in memory) print("=" * 80) -print("در حال بارگذاری Text Encoder...") +print("Loading Gemma Text Encoder...") print("=" * 80) checkpoint_path = get_hub_or_local_checkpoint(DEFAULT_REPO_ID, DEFAULT_CHECKPOINT_FILENAME) gemma_local_path = download_gemma_model(DEFAULT_GEMMA_REPO_ID) device = "cuda" +print(f"Initializing text encoder with:") +print(f" checkpoint_path={checkpoint_path}") +print(f" gemma_root={gemma_local_path}") +print(f" device={device}") + model_ledger = ModelLedger( dtype=torch.bfloat16, device=device, @@ -188,177 +218,370 @@ model_ledger = ModelLedger( gemma_root_path=DEFAULT_GEMMA_REPO_ID, local_files_only=False ) + +# Load text encoder once and keep it in memory text_encoder = model_ledger.text_encoder() -print("Text encoder آماده است!") - - -# --- PART 7: CONTROLNET & POSE PRE-PROCESSORS --- -canny_processor = CannyDetector() -depth_processor = MidasDetector.from_pretrained("lllyasviel/Annotators").to("cuda") - -def process_video_for_pose(frames, width: int, height: int): - pose_processor = DwposeDetector.from_pretrained_default() - if not frames: return [] - pose_frames = [] - for frame in frames: - pil = Image.fromarray(frame.astype(np.uint8)).convert("RGB") - pose_img = pose_processor(pil, include_body=True, include_hand=True, include_face=True) - if not isinstance(pose_img, Image.Image): - pose_img = Image.fromarray(pose_img.astype(np.uint8)) - pose_img = pose_img.convert("RGB").resize((width, height), Image.BILINEAR) - pose_np = np.array(pose_img).astype(np.float32) / 255.0 - pose_frames.append(pose_np) - return pose_frames - - -# --- PART 8: VIDEO PRE-PROCESSING (POSE/DEPTH/CANNY) --- -def preprocess_video_to_pose_mp4(video_path: str, width: int, height: int, fps: float): - frames = load_video_frames(video_path) - pose_frames = process_video_for_pose(frames, width=width, height=height) - tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) - tmp.close() - return write_video_mp4(pose_frames, fps=fps, out_path=tmp.name) - -def process_video_for_depth(frames, width: int, height: int): - if not frames: return [] - detect_resolution = max(frames[0].shape[0], frames[0].shape[1]) - image_resolution = max(width, height) - depth_frames = [] - for frame in frames: - depth = depth_processor(frame, detect_resolution=detect_resolution, image_resolution=image_resolution, output_type="np") - if depth.ndim == 2: depth = np.stack([depth, depth, depth], axis=-1) - elif depth.shape[-1] == 1: depth = np.repeat(depth, 3, axis=-1) - depth_frames.append(depth) - return depth_frames - -def preprocess_video_to_depth_mp4(video_path: str, width: int, height: int, fps: float): - frames = load_video_frames(video_path) - depth_frames = process_video_for_depth(frames, width=width, height=height) - tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) - tmp.close() - return write_video_mp4(depth_frames, fps=fps, out_path=tmp.name) - -def process_video_for_canny(frames, width: int, height: int, low_threshold=20, high_threshold=60): - if not frames: return [] - detect_resolution = max(frames[0].shape[0], frames[0].shape[1]) - image_resolution = max(width, height) - canny_frames = [] - for frame in frames: - canny = canny_processor(frame, low_threshold=low_threshold, high_threshold=high_threshold, - detect_resolution=detect_resolution, image_resolution=image_resolution, output_type="np") - canny_frames.append(canny) - return canny_frames - - -# --- PART 9: CONDITIONING VIDEO PREPARATION --- + +print("=" * 80) +print("Text encoder loaded and ready!") +print("=" * 80) + +def on_lora_change(selected: str): + # Only Detailer might need handling, but we simplified motion control out + needs_video = selected in {"Detailer"} + return ( + selected, + gr.update(visible=not needs_video, value=None if needs_video else None), + gr.update(visible=needs_video, value=None if not needs_video else None), + ) + + +#################################################################################################### +### PART 6: Video/Frame Utilities +#################################################################################################### +def load_video_frames(video_path: str): + """Return list of frames as numpy arrays (H,W,3) uint8.""" + frames = [] + with imageio.get_reader(video_path) as reader: + for frame in reader: + frames.append(frame) + return frames + +def write_video_mp4(frames_float_01, fps: float, out_path: str): + """Write frames in float [0..1] to mp4 as uint8.""" + frames_uint8 = [(f * 255).astype(np.uint8) for f in frames_float_01] + + # PyAV backend doesn't support `quality=...` + with imageio.get_writer(out_path, fps=fps, macro_block_size=1) as writer: + for fr in frames_uint8: + writer.append_data(fr) + return out_path + +import json + +def probe_video_duration_seconds(video_path: str) -> float: + """Return duration in seconds using ffprobe.""" + cmd = [ + "ffprobe", "-v", "error", + "-select_streams", "v:0", + "-show_entries", "format=duration", + "-of", "json", + video_path, + ] + out = subprocess.check_output(cmd).decode("utf-8") + data = json.loads(out) + dur = float(data["format"]["duration"]) + return dur + + +#################################################################################################### +### PART 7: FFmpeg Utils & Frame Prep +#################################################################################################### +def trim_video_to_seconds_ffmpeg(video_path: str, target_seconds: float, fps: float = None) -> str: + """ + Trim video to [0, target_seconds]. Re-encode for accuracy & compatibility. + If fps is provided, also normalize fps. + Returns new temp mp4 path. + """ + target_seconds = max(0.01, float(target_seconds)) + + out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name + + vf = [] + if fps is not None: + vf.append(f"fps={float(fps)}") + vf_str = ",".join(vf) if vf else None + + cmd = ["ffmpeg", "-y", "-v", "error"] + + # Accurate trim: use -t and re-encode. + cmd += ["-i", video_path, "-t", f"{target_seconds:.6f}"] + + if vf_str: + cmd += ["-vf", vf_str] + + # Safe default encode + cmd += [ + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-preset", "veryfast", "-crf", "18", + "-an", # conditioning video doesn't need audio + out_path + ] + + subprocess.check_call(cmd) + return out_path + +def extract_first_frame_png(video_path: str) -> str: + """Extract first frame as png; returns png path.""" + out_path = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name + cmd = [ + "ffmpeg", "-y", "-v", "error", + "-i", video_path, + "-frames:v", "1", + out_path + ] + subprocess.check_call(cmd) + return out_path + def _coerce_video_path(video_path: Any) -> str: - if isinstance(video_path, tuple) and len(video_path) > 0: video_path = video_path[0] - if isinstance(video_path, dict): video_path = video_path.get("name") or video_path.get("path") + if isinstance(video_path, tuple) and len(video_path) > 0: + video_path = video_path[0] + if isinstance(video_path, dict): + video_path = video_path.get("name") or video_path.get("path") if not isinstance(video_path, (str, bytes, os.PathLike)): raise TypeError(f"video_path must be a path-like, got {type(video_path)}: {video_path}") return os.fspath(video_path) -def valid_1_plus_8k(n: int) -> int: - if n <= 0: return 0 - return 1 + 8 * ((n - 1) // 8) - -def prepare_conditioning_video_mp4_no_pad(video_path: Any, duration_frames: int, target_fps: float) -> tuple[str, str, int]: - video_path = _coerce_video_path(video_path) - frames = load_video_frames(video_path) - if not frames: raise ValueError("فریمهای ویدیو بارگذاری نشد.") - n_src = min(len(frames), duration_frames) - n_used = valid_1_plus_8k(n_src) - if n_used == 0: raise ValueError(f"ویدیو خیلی کوتاه است: {n_src} فریم") - frames = frames[:n_used] - first_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name - Image.fromarray(frames[0]).save(first_png) - frames_float = [f.astype(np.float32) / 255.0 for f in frames] - cond_mp4 = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name - write_video_mp4(frames_float, fps=target_fps, out_path=cond_mp4) - return cond_mp4, first_png, n_used +#################################################################################################### +### PART 8: Prompt Encoding Logic +#################################################################################################### def encode_text_simple(text_encoder, prompt: str): + """Simple text encoding without using pipeline_utils.""" v_context, a_context, _ = text_encoder(prompt) return v_context, a_context @spaces.GPU() -def encode_prompt(prompt: str, enhance_prompt: bool = True, input_image=None, seed: int = 42, negative_prompt: str = ""): +def encode_prompt( + prompt: str, + enhance_prompt: bool = True, + input_image=None, # this is now filepath (string) or None + seed: int = 42, + negative_prompt: str = "" +): start_time = time.time() try: final_prompt = prompt if enhance_prompt: - final_prompt = generate_enhanced_prompt(text_encoder=text_encoder, prompt=prompt, - image_path=input_image, seed=seed) + final_prompt = generate_enhanced_prompt( + text_encoder=text_encoder, + prompt=prompt, + image_path=input_image if input_image is not None else None, + seed=seed, + ) + with torch.inference_mode(): video_context, audio_context = encode_text_simple(text_encoder, final_prompt) - + + video_context_negative = None + audio_context_negative = None + if negative_prompt: + video_context_negative, audio_context_negative = encode_text_simple(text_encoder, negative_prompt) + + # IMPORTANT: return tensors directly (no torch.save) embedding_data = { "video_context": video_context.detach().cpu(), "audio_context": audio_context.detach().cpu(), "prompt": final_prompt, + "original_prompt": prompt, } - status = f"✓ انکود شده در {time.time() - start_time:.2f} ثانیه" + if video_context_negative is not None: + embedding_data["video_context_negative"] = video_context_negative + embedding_data["audio_context_negative"] = audio_context_negative + embedding_data["negative_prompt"] = negative_prompt + + elapsed_time = time.time() - start_time + if torch.cuda.is_available(): + allocated = torch.cuda.memory_allocated() / 1024**3 + peak = torch.cuda.max_memory_allocated() / 1024**3 + status = f"✓ Encoded in {elapsed_time:.2f}s | VRAM: {allocated:.2f}GB allocated, {peak:.2f}GB peak" + else: + status = f"✓ Encoded in {elapsed_time:.2f}s (CPU mode)" + return embedding_data, final_prompt, status + except Exception as e: - return None, prompt, f"خطا: {str(e)}" + import traceback + error_msg = f"Error: {str(e)}\n{traceback.format_exc()}" + print(error_msg) + return None, prompt, error_msg +# Default prompt from docstring example +DEFAULT_PROMPT = "An astronaut hatches from a fragile egg on the surface of the Moon, the shell cracking and peeling apart in gentle low-gravity motion. Fine lunar dust lifts and drifts outward with each movement, floating in slow arcs before settling back onto the ground. The astronaut pushes free in a deliberate, weightless motion, small fragments of the egg tumbling and spinning through the air. In the background, the deep darkness of space subtly shifts as stars glide with the camera's movement, emphasizing vast depth and scale. The camera performs a smooth, cinematic slow push-in, with natural parallax between the foreground dust, the astronaut, and the distant starfield. Ultra-realistic detail, physically accurate low-gravity motion, cinematic lighting, and a breath-taking, movie-like shot." -# --- PART 10: PIPELINE INITIALIZATION (DiT & LoRAs) --- -print("=" * 80) -print("در حال بارگذاری پایپلاین LTX-2 Distilled...") -print("=" * 80) +#################################################################################################### +### PART 9: Pipeline Constants & Checkpoint Paths +#################################################################################################### +# HuggingFace Hub defaults +DEFAULT_REPO_ID = "Lightricks/LTX-2" +DEFAULT_CHECKPOINT_FILENAME = "ltx-2-19b-dev.safetensors" DEFAULT_DISTILLED_LORA_FILENAME = "ltx-2-19b-distilled-lora-384.safetensors" DEFAULT_SPATIAL_UPSAMPLER_FILENAME = "ltx-2-spatial-upscaler-x2-1.0.safetensors" -spatial_upsampler_path = get_hub_or_local_checkpoint(DEFAULT_REPO_ID, DEFAULT_SPATIAL_UPSAMPLER_FILENAME) -distilled_lora_path = get_hub_or_local_checkpoint(DEFAULT_REPO_ID, DEFAULT_DISTILLED_LORA_FILENAME) +def get_hub_or_local_checkpoint_pipeline(repo_id: Optional[str] = None, filename: Optional[str] = None): + """Download from HuggingFace Hub or use local checkpoint.""" + if repo_id is None and filename is None: + raise ValueError("Please supply at least one of `repo_id` or `filename`") + + if repo_id is not None: + if filename is None: + raise ValueError("If repo_id is specified, filename must also be specified.") + print(f"Downloading {filename} from {repo_id}...") + ckpt_path = hf_hub_download(repo_id=repo_id, filename=filename) + print(f"Downloaded to {ckpt_path}") + else: + ckpt_path = filename + + return ckpt_path + + +# Initialize pipeline at startup +print("=" * 80) +print("Loading LTX-2 Distilled pipeline...") +print("=" * 80) + +checkpoint_path = get_hub_or_local_checkpoint_pipeline(DEFAULT_REPO_ID, DEFAULT_CHECKPOINT_FILENAME) +spatial_upsampler_path = get_hub_or_local_checkpoint_pipeline(DEFAULT_REPO_ID, DEFAULT_SPATIAL_UPSAMPLER_FILENAME) + +print(f"Initializing pipeline with:") +print(f" checkpoint_path={checkpoint_path}") +print(f" spatial_upsampler_path={spatial_upsampler_path}") + +distilled_lora_path = get_hub_or_local_checkpoint_pipeline( + DEFAULT_REPO_ID, + DEFAULT_DISTILLED_LORA_FILENAME, +) + +static_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-LoRA-Camera-Control-Static", + "ltx-2-19b-lora-camera-control-static.safetensors", +) +dolly_in_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In", + "ltx-2-19b-lora-camera-control-dolly-in.safetensors", +) +dolly_out_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Out", + "ltx-2-19b-lora-camera-control-dolly-out.safetensors", +) +dolly_left_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Left", + "ltx-2-19b-lora-camera-control-dolly-left.safetensors", +) +dolly_right_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Right", + "ltx-2-19b-lora-camera-control-dolly-right.safetensors", +) +jib_down_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Down", + "ltx-2-19b-lora-camera-control-jib-down.safetensors", +) +jib_up_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Up", + "ltx-2-19b-lora-camera-control-jib-up.safetensors", +) + +detailer_lora_path = get_hub_or_local_checkpoint_pipeline( + "Lightricks/LTX-2-19b-IC-LoRA-Detailer", + "ltx-2-19b-ic-lora-detailer.safetensors", +) -# دانلود و تنظیم مسیر LoRAها -def get_lora(repo, name): return get_hub_or_local_checkpoint(repo, name) +#################################################################################################### +### PART 10: LoRA Setup & Configuration +#################################################################################################### +# Load distilled LoRA as a regular LoRA loras = [ - LoraPathStrengthAndSDOps(distilled_lora_path, 0.6, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-LoRA-Camera-Control-Static", "ltx-2-19b-lora-camera-control-static.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-IC-LoRA-Detailer", "ltx-2-19b-ic-lora-detailer.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In", "ltx-2-19b-lora-camera-control-dolly-in.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Out", "ltx-2-19b-lora-camera-control-dolly-out.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Left", "ltx-2-19b-lora-camera-control-dolly-left.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Right", "ltx-2-19b-lora-camera-control-dolly-right.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Down", "ltx-2-19b-lora-camera-control-jib-down.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Up", "ltx-2-19b-lora-camera-control-jib-up.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), - LoraPathStrengthAndSDOps(get_lora("Lightricks/LTX-2-19b-IC-LoRA-Pose-Control", "ltx-2-19b-ic-lora-pose-control.safetensors"), DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + # --- fused / base behavior --- + LoraPathStrengthAndSDOps( + path=distilled_lora_path, + strength=0.6, + sd_ops=LTXV_LORA_COMFY_RENAMING_MAP, + ), + LoraPathStrengthAndSDOps(static_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + LoraPathStrengthAndSDOps(detailer_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + LoraPathStrengthAndSDOps(dolly_in_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + LoraPathStrengthAndSDOps(dolly_out_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + LoraPathStrengthAndSDOps(dolly_left_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + LoraPathStrengthAndSDOps(dolly_right_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + LoraPathStrengthAndSDOps(jib_down_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), + LoraPathStrengthAndSDOps(jib_up_lora_path, DEFAULT_LORA_STRENGTH, LTXV_LORA_COMFY_RENAMING_MAP), +] + +# Runtime-toggle LoRAs (exclude fused distilled at index 0) +VISIBLE_RUNTIME_LORA_CHOICES = [ + ("No LoRA", -1), + ("Static", 0), + ("Detailer", 1), + ("Zoom In", 2), + ("Zoom Out", 3), + ("Slide Left", 4), + ("Slide Right", 5), + ("Slide Down", 6), + ("Slide Up", 7), ] -# لیست انتخابی برای UI (فارسی به مقدار عددی) RUNTIME_LORA_CHOICES = [ - ("هیچکدام", -1), ("ثابت (Static)", 0), ("جزئیاتدهنده (Detailer)", 1), - ("زوم به داخل", 2), ("زوم به عقب", 3), ("حرکت به چپ", 4), - ("حرکت به راست", 5), ("حرکت به پایین", 6), ("حرکت به بالا", 7), ("ژست (Pose)", 8), + ("No LoRA", -1), + ("Static", 0), + ("Detailer", 1), + ("Zoom In", 2), + ("Zoom Out", 3), + ("Slide Left", 4), + ("Slide Right", 5), + ("Slide Down", 6), + ("Slide Up", 7), ] + +#################################################################################################### +### PART 11: Pipeline Initialization +#################################################################################################### +# Initialize pipeline WITHOUT text encoder (gemma_root=None) +# Text encoding will be done by external space pipeline = DistilledPipeline( - device=torch.device("cuda"), checkpoint_path=checkpoint_path, spatial_upsampler_path=spatial_upsampler_path, - gemma_root=None, loras=loras, fp8transformer=False, local_files_only=False, + device=torch.device("cuda"), + checkpoint_path=checkpoint_path, + spatial_upsampler_path=spatial_upsampler_path, + gemma_root=None, # No text encoder in this space + loras=loras, + fp8transformer=False, + local_files_only=False, ) + pipeline._video_encoder = pipeline.model_ledger.video_encoder() pipeline._transformer = pipeline.model_ledger.transformer() -print("پایپلاین آماده است!") +print("=" * 80) +print("Pipeline fully loaded and ready!") +print("=" * 80) -# --- PART 11: CUSTOM COMPONENT - RadioAnimated (JS/HTML) --- + +#################################################################################################### +### PART 12: Custom Component - RadioAnimated +#################################################################################################### class RadioAnimated(gr.HTML): + """ + Animated segmented radio (like iOS pill selector). + Outputs: selected option string, e.g. "768x512" + """ def __init__(self, choices, value=None, **kwargs): - if value is None: value = choices[0] - uid = uuid.uuid4().hex[:8] + if not choices or len(choices) < 2: + raise ValueError("RadioAnimated requires at least 2 choices.") + if value is None: + value = choices[0] + + uid = uuid.uuid4().hex[:8] # unique per instance group_name = f"ra-{uid}" + inputs_html = "\n".join( - f'' - f'' + f""" + + + """ for i, c in enumerate(choices) ) - html_template = f'
متن بنویسید، تصویر بدهید و ویدیو تحویل بگیرید.
+css += """ + /* --- prompt box --- */ + .ds-prompt{ + width: 100%; + max-width: 720px; + margin-top: 3px; + } + + .ds-textarea{ + width: 100%; + box-sizing: border-box; + background: #2b2b2b; + color: rgba(255,255,255,0.9); + border: 1px solid rgba(255,255,255,0.12); + border-radius: 14px; + padding: 14px 16px; + outline: none; + font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial; + font-size: 15px; + line-height: 1.35; + resize: none; + min-height: 210px; + max-height: 210px; + overflow-y: auto; + + /* IMPORTANT: space for the footer controls */ + padding-bottom: 72px; + } + + + .ds-card{ + width: 100%; + max-width: 720px; + margin: 0 auto; + } + .ds-top{ + position: relative; + } + + /* Make room for footer inside textarea */ + .ds-textarea{ + padding-bottom: 72px; + } + + /* Footer positioning */ + .ds-footer{ + position: absolute; + right: 12px; + bottom: 10px; + display: flex; + gap: 8px; + align-items: center; + justify-content: flex-end; + z-index: 3; + } + + /* Smaller pill buttons inside footer */ + .ds-footer .cd-trigger{ + min-height: 32px; + padding: 6px 10px; + font-size: 12px; + gap: 6px; + border-radius: 9999px; + } + .ds-footer .cd-trigger-icon, + .ds-footer .cd-icn{ + width: 14px; + height: 14px; + } + .ds-footer .cd-trigger-icon svg, + .ds-footer .cd-icn svg{ + width: 14px; + height: 14px; + } + .ds-footer .cd-caret{ + font-size: 11px; + } + + /* Bottom safe area bar (optional but looks nicer) */ + .ds-top::after{ + content: ""; + position: absolute; + left: 1px; + right: 1px; + bottom: 1px; + height: 56px; + background: #2b2b2b; + border-bottom-left-radius: 13px; + border-bottom-right-radius: 13px; + pointer-events: none; + z-index: 2; + } + + """ + +css += """ + /* ---- camera dropdown ---- */ + + /* 1) Fix overlap: make the Gradio HTML block shrink-to-fit when it contains a CameraDropdown. + Gradio uses .gr-html for HTML components in most versions; older themes sometimes use .gradio-html. + This keeps your big header HTML unaffected because it doesn't contain .cd-wrap. + */ + + /* 2) Actual dropdown layout */ + .cd-wrap{ + position: relative; + display: inline-block; + } + + /* 3) Match RadioAnimated pill size/feel */ + .cd-trigger{ + margin-top: 2px; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 10px; + + border: none; + + box-sizing: border-box; + padding: 10px 18px; + min-height: 52px; + line-height: 1.2; + + border-radius: 9999px; + background: #0b0b0b; + + font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial; + font-size: 14px; + + /* ✅ match .ra-label exactly */ + color: rgba(255,255,255,0.7) !important; + font-weight: 600 !important; + + cursor: pointer; + user-select: none; + white-space: nowrap; + } + + /* Ensure inner spans match too */ + .cd-trigger .cd-trigger-text, + .cd-trigger .cd-caret{ + color: rgba(255,255,255,0.7) !important; + } + + /* keep caret styling */ + .cd-caret{ + opacity: 0.8; + font-weight: 900; + } + + /* 4) Ensure menu overlays neighbors and isn't clipped */ + /* Move dropdown a tiny bit up (closer to the trigger) */ + .cd-menu{ + position: absolute; + top: calc(100% + 4px); /* was +10px */ + left: 0; + + min-width: 240px; + background: #2b2b2b; + border: 1px solid rgba(255,255,255,0.14); + border-radius: 14px; + box-shadow: 0 18px 40px rgba(0,0,0,0.35); + padding: 10px; + + opacity: 0; + transform: translateY(-6px); + pointer-events: none; + transition: opacity 160ms ease, transform 160ms ease; + + z-index: 9999; + } + + .cd-title{ + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + + color: rgba(255,255,255,0.45); /* 👈 muted grey */ + margin-bottom: 6px; + padding: 0 6px; + pointer-events: none; /* title is non-interactive */ + } + + + .cd-menu.open{ + opacity: 1; + transform: translateY(0); + pointer-events: auto; + } + + .cd-items{ + display: flex; + flex-direction: column; + gap: 0px; /* tighter, more like a native menu */ + } + + /* Items: NO "boxed" buttons by default */ + .cd-item{ + width: 100%; + text-align: left; + border: none; + background: transparent; /* ✅ removes box look */ + color: rgba(255,255,255,0.92); + padding: 8px 34px 8px 12px; /* right padding leaves room for tick */ + border-radius: 10px; /* only matters on hover */ + cursor: pointer; + + font-size: 14px; + font-weight: 700; + + position: relative; + transition: background 120ms ease; + } + + /* “Box effect” only on hover (not always) */ + .cd-item:hover{ + background: rgba(255,255,255,0.08); + } + + /* Tick on the right ONLY on hover */ + .cd-item::after{ + content: "✓"; + position: absolute; + right: 12px; + top: 50%; + transform: translateY(-50%); + opacity: 0; /* hidden by default */ + transition: opacity 120ms ease; + color: rgba(255,255,255,0.9); + font-weight: 900; + } + + /* show tick ONLY for selected item */ + .cd-item[data-selected="true"]::after{ + opacity: 1; + } + + /* keep hover box effect, but no tick change */ + .cd-item:hover{ + background: rgba(255,255,255,0.08); + } + + + /* Kill any old “selected” styling just in case */ + .cd-item.selected{ + background: transparent !important; + border: none !important; + } + + + """ + +css += """ +/* icons in dropdown items */ +.cd-item{ + display: flex; + align-items: center; + gap: 10px; +} +.cd-icn{ + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + flex: 0 0 18px; +} +.cd-label{ + flex: 1; +} + +/* ========================= + FIX: prompt border + scrollbar bleed + ========================= */ + +/* Put the border + background on the wrapper, not the textarea */ +.ds-top{ + position: relative; + background: #2b2b2b; + border: 1px solid rgba(255,255,255,0.12); + border-radius: 14px; + overflow: hidden; /* ensures the footer bar is clipped to rounded corners */ +} + +/* Make textarea "transparent" so wrapper owns the border/background */ +.ds-textarea{ + background: transparent !important; + border: none !important; + border-radius: 0 !important; /* wrapper handles radius */ + outline: none; + + /* keep your spacing */ + padding: 14px 16px; + padding-bottom: 72px; /* room for footer */ + width: 100%; + box-sizing: border-box; + + /* keep scroll behavior */ + overflow-y: auto; + + /* prevent scrollbar bleed by hiding native scrollbar */ + scrollbar-width: none; /* Firefox */ +} +.ds-textarea::-webkit-scrollbar{ /* Chrome/Safari */ + width: 0; + height: 0; +} + +/* Safe-area bar: now it matches perfectly because it's inside the same bordered wrapper */ +.ds-top::after{ + content: ""; + position: absolute; + left: 0; + right: 0; + bottom: 0; + height: 56px; + background: #2b2b2b; + pointer-events: none; + z-index: 2; +} + +/* Footer above the bar */ +.ds-footer{ + position: absolute; + right: 12px; + bottom: 10px; + display: flex; + gap: 8px; + align-items: center; + justify-content: flex-end; + z-index: 3; +} + +/* Ensure textarea content sits below overlays */ +.ds-textarea{ + position: relative; + z-index: 1; +} + +/* ===== FIX dropdown menu being clipped/behind ===== */ + +/* Let the dropdown menu escape the prompt wrapper */ +.ds-top{ + overflow: visible !important; /* IMPORTANT: do not clip the menu */ +} + +/* Keep the rounded "safe area" look without clipping the menu */ +.ds-top::after{ + left: 0 !important; + right: 0 !important; + bottom: 0 !important; + border-bottom-left-radius: 14px !important; + border-bottom-right-radius: 14px !important; +} + +/* Ensure the footer stays above the safe-area bar */ +.ds-footer{ + z-index: 20 !important; +} + +/* Make sure the opened menu is above EVERYTHING */ +.ds-footer .cd-menu{ + z-index: 999999 !important; +} + +/* Sometimes Gradio/columns/cards create stacking contexts; + force the whole prompt card above nearby panels */ +.ds-card{ + position: relative; + z-index: 50; +} + +/* --- Fix focus highlight shape (make it match rounded container) --- */ + +/* Kill any theme focus ring on the textarea itself */ +.ds-textarea:focus, +.ds-textarea:focus-visible{ + outline: none !important; + box-shadow: none !important; +} + +/* Optional: if some themes apply it even when not focused */ +.ds-textarea{ + outline: none !important; +} + +/* Apply the focus ring to the rounded wrapper instead */ +.ds-top:focus-within{ + border-color: rgba(255,255,255,0.22) !important; + box-shadow: 0 0 0 3px rgba(255,255,255,0.06) !important; + border-radius: 14px !important; +} + +/* If you see any tiny square corners, ensure the wrapper clips its own shadow properly */ +.ds-top{ + border-radius: 14px !important; +} + +/* ========================= + CameraDropdown: force readable menu text in BOTH themes + ========================= */ + +/* Menu surface */ +.cd-menu{ + background: #2b2b2b !important; + border: 1px solid rgba(255,255,255,0.14) !important; +} + +/* Title */ +.cd-title{ + color: rgba(255,255,255,0.55) !important; +} + +/* Items + all descendants (fixes spans / inherited theme colors) */ +.cd-item, +.cd-item *{ + color: rgba(255,255,255,0.92) !important; +} + +/* Hover state */ +.cd-item:hover{ + background: rgba(255,255,255,0.10) !important; +} + +/* Checkmark */ +.cd-item::after{ + color: rgba(255,255,255,0.92) !important; +} + +/* (Optional) make sure the trigger stays readable too */ +.cd-trigger, +.cd-trigger *{ + color: rgba(255,255,255,0.75) !important; +} + +/* ---- preset gallery ---- */ +.pg-wrap{ + width: 100%; + max-width: 1100px; + margin: 18px auto 0 auto; +} +.pg-title{ + text-align: center; + margin-bottom: 14px; +} +.pg-h1{ + font-size: 34px; + font-weight: 800; + line-height: 1.1; + + /* ✅ theme-aware */ + color: var(--body-text-color); +} +.pg-h2{ + font-size: 14px; + font-weight: 600; + color: var(--body-text-color-subdued); + margin-top: 6px; +} + +.pg-grid{ + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); /* 3 per row */ + gap: 18px; +} + +.pg-card{ + border: none; + background: transparent; + padding: 0; + cursor: pointer; + border-radius: 12px; + overflow: hidden; + position: relative; + transform: translateZ(0); +} + +.pg-img{ + width: 100%; + height: 220px; /* adjust to match your look */ + object-fit: cover; + display: block; + border-radius: 12px; + transition: transform 160ms ease, filter 160ms ease, opacity 160ms ease; +} + +/* hover: slight zoom on hovered card */ +.pg-card:hover .pg-img{ + transform: scale(1.02); +} + +/* dim others while hovering */ +.pg-card[data-dim="true"] .pg-img{ + opacity: 0.35; + filter: saturate(0.9); +} + +/* keep hovered/active crisp */ +.pg-card[data-active="true"] .pg-img{ + opacity: 1.0; + filter: none; +} + + +""" + + +css += """ +/* ---- AudioDropUpload ---- */ +.aud-wrap{ + width: 100%; + max-width: 720px; +} +.aud-drop{ + border: 2px dashed var(--body-text-color-subdued); + border-radius: 16px; + padding: 18px; + text-align: center; + cursor: pointer; + user-select: none; + color: var(--body-text-color); + background: var(--block-background-fill); +} +.aud-drop.dragover{ + border-color: rgba(255,255,255,0.35); + background: rgba(255,255,255,0.06); +} +.aud-hint{ + color: var(--body-text-color-subdued); + font-size: 0.95rem; + margin-top: 6px; +} +/* pill row like your other controls */ +.aud-row{ + display: none; + align-items: center; + gap: 10px; + background: #0b0b0b; + border-radius: 9999px; + padding: 8px 10px; +} +.aud-player{ + flex: 1; + width: 100%; + height: 34px; + border-radius: 9999px; +} +.aud-remove{ + appearance: none; + border: none; + background: transparent; + color: rgba(255,255,255); + cursor: pointer; + width: 36px; + height: 36px; + border-radius: 9999px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0; + transition: background 120ms ease, color 120ms ease, opacity 120ms ease; + opacity: 0.9; + flex: 0 0 auto; +} +.aud-remove:hover{ + background: rgba(255,255,255,0.08); + color: rgb(255,255,255); + opacity: 1; +} +.aud-filelabel{ + margin: 10px 6px 0; + color: var(--body-text-color-subdued); + font-size: 0.95rem; + display: none; +} +#audio_input_hidden { display: none !important; } +""" + + +def apply_example(idx: str): + idx = int(idx) + + # Read the example row from your list + img, prompt_txt, cam, res, mode, vid, aud, end_img = examples_list[idx] + + img_path = img if img else None + vid_path = vid if vid else None + aud_path = aud if aud else None + + input_image_update = img_path + prompt_update = prompt_txt + camera_update = cam + resolution_update = res + mode_update = mode + video_update = gr.update(value=vid_path, visible=(mode == "Motion Control")) + audio_update = aud_path + end_image = end_img + + return ( + input_image_update, + prompt_update, + camera_update, + resolution_update, + mode_update, + video_update, + audio_update, + audio_update, + end_image, + ) + + +#################################################################################################### +### PART 20: Gradio UI Layout & Launch +#################################################################################################### +with gr.Blocks(title="LTX-2 Video Distilled 🎥🔈") as demo: + + gr.HTML( + """ ++ LTX-2 Distilled DiT-based audio-video foundation model +
+ + [model] + +