import os os.environ.setdefault("HF_HOME", "/tmp/.cache/huggingface") os.environ.setdefault("HF_MODULES_CACHE", "/tmp/hf_modules") os.environ.setdefault("MPLCONFIGDIR", "/tmp/matplotlib") os.environ.setdefault("GRADIO_SSR_MODE", "false") os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") for _path in ( os.environ["HF_HOME"], os.environ["HF_MODULES_CACHE"], os.environ["MPLCONFIGDIR"], ): os.makedirs(_path, exist_ok=True) import random import gc import tempfile import time import gradio as gr import imageio.v3 as iio import numpy as np import spaces import torch from huggingface_hub import hf_hub_download from PIL import Image, ImageOps from safetensors.torch import load_file from diffusers import LTX2InContextPipeline, LTX2LatentUpsamplePipeline from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES from diffusers.utils import encode_video, load_video BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers" EDIT_REPO = "Alissonerdx/EditAnything" MOTION_LORA = "edit_anything_30k_v0.1_motion_transfer_r128.safetensors" PROMPT_LORA = "edit_anything_v1.1_r256.safetensors" UPSAMPLER_REPO = "dg845/LTX-2.3-Spatial-Upsampler-Diffusers" FPS = 24 NUM_STEPS = len(DISTILLED_SIGMA_VALUES) MAX_SEED = np.iinfo(np.int32).max HF_TOKEN = os.environ.get("HF_TOKEN") MODE_MOTION = "Motion Transfer (v0.1)" MODE_PROMPT = "Prompt Edit (v1.1)" MODE_REF = "Ref V2V (experimental, not enabled)" ADAPTERS = { MODE_MOTION: ("motion_v01_r128", MOTION_LORA), MODE_PROMPT: ("prompt_v11_r256", PROMPT_LORA), } RES_PRESETS = { "Fast (768x448)": (768, 448), "Quality (960x544)": (960, 544), } FRAME_CHOICES = [49, 73, 97, 121] EXAMPLE_DIR = "examples" UNUSED_FIRST_FRAME_EXAMPLE = f"{EXAMPLE_DIR}/motion_edited_first_frame.png" PROMPT_EDIT_EXAMPLES = [ [ MODE_PROMPT, f"{EXAMPLE_DIR}/prompt_add_source.mp4", "Add", "Add a golden retriever sitting on the grass beside the suitcase, to the right of the woman on the park bench.", "", ], [ MODE_PROMPT, f"{EXAMPLE_DIR}/prompt_remove_source.mp4", "Remove", "Remove the woman walking on the path.", "", ], [ MODE_PROMPT, f"{EXAMPLE_DIR}/prompt_replace_source.mp4", "Replace", "Replace the large red triangular sculpture in the background with a stone fountain spraying water in the plaza.", "", ], [ MODE_PROMPT, f"{EXAMPLE_DIR}/prompt_style_source.mp4", "Style", "", "Watercolor Painting", ], ] MOTION_TRANSFER_EXAMPLES = [ [ MODE_MOTION, f"{EXAMPLE_DIR}/motion_guide_source.mp4", f"{EXAMPLE_DIR}/motion_edited_first_frame.png", "Change the dancer into a copper-red-haired dancer wearing a loose white sweater and dark pants on the same rocky beach.", "", ], ] APP_THEME = gr.themes.Soft( primary_hue="orange", secondary_hue="sky", neutral_hue="slate", spacing_size="sm", radius_size="sm", text_size="sm", font=gr.themes.GoogleFont("Inter"), ).set( button_primary_background_fill="#f97316", button_primary_background_fill_hover="#ea580c", button_primary_text_color="#ffffff", block_border_width="1px", block_shadow="none", ) CUSTOM_CSS = """ :root { --ea-max-width: 1280px; } html, body, gradio-app { overflow-x: hidden !important; } .gradio-container, .gradio-container * { box-sizing: border-box; } .gradio-container { width: min(var(--ea-max-width), calc(100vw - 32px)) !important; max-width: var(--ea-max-width) !important; margin: 0 auto !important; color: #111827; min-width: 0 !important; } .ea-main-row { align-items: flex-start !important; gap: 0.75rem !important; width: 100% !important; min-width: 0 !important; } .ea-main-row > *, .ea-input-column, .ea-output-column { min-width: 0 !important; max-width: 100% !important; } .ea-header { margin: 0 0 14px; } .ea-header h1 { font-size: clamp(1.45rem, 2vw, 2rem); line-height: 1.12; margin-bottom: 0.35rem; letter-spacing: 0; } .ea-header p { color: #64748b; margin: 0; } .ea-column-title h2, .ea-column-title h3 { font-size: 0.95rem; line-height: 1.25; margin: 0 0 0.25rem; } .ea-hint p, .ea-task-hint p { color: #64748b; font-size: 0.9rem; margin: 0; } .ea-output-column { position: sticky; top: 12px; align-self: start; } .ea-generate button { min-height: 48px; font-weight: 700; letter-spacing: 0; } .edit-type-selector .wrap { display: flex !important; gap: 0.45rem; flex-wrap: wrap !important; } .edit-type-selector [role="radiogroup"] { display: flex !important; flex-wrap: wrap !important; gap: 0.45rem !important; } .edit-type-selector label { flex: 1 1 120px; min-width: 0; border-radius: 8px !important; border: 1px solid #cbd5e1 !important; padding: 0.5rem 0.72rem !important; background: #ffffff !important; } .edit-type-selector label:has(input:checked) { border-color: #f97316 !important; background: #fff7ed !important; color: #9a3412 !important; font-weight: 700; } .ea-examples table { font-size: 0.82rem; } .ea-examples .table-wrap { max-height: 320px; overflow: auto; max-width: 100%; } footer { display: none !important; } @media (max-width: 1000px) { .gradio-container { padding-top: 52px !important; padding-left: 14px !important; padding-right: 14px !important; overflow-x: hidden !important; } .ea-output-column { position: static; } } @media (max-width: 760px) { .gradio-container { width: 100% !important; max-width: 100vw !important; padding: 56px 16px 24px !important; } .ea-header { padding-top: 0 !important; max-width: 100% !important; } .ea-main-row { display: flex !important; flex-direction: column !important; gap: 0.85rem !important; } .ea-main-row > *, .ea-input-column, .ea-output-column, .gradio-container .block, .gradio-container .form, .gradio-container .panel, .gradio-container .tabs, .gradio-container .tabitem { width: 100% !important; max-width: 100% !important; min-width: 0 !important; flex: 1 1 auto !important; } .ea-output-column { margin-top: 0.25rem; } .edit-type-selector .wrap, .edit-type-selector [role="radiogroup"] { width: 100% !important; max-width: 100% !important; } .edit-type-selector label { flex: 1 1 100% !important; max-width: 100% !important; min-height: 46px; display: flex !important; align-items: center; } .ea-generate button { min-height: 52px; } .ea-examples .table-wrap { overflow-x: auto; } .ea-examples table { min-width: 560px; } } @media (max-width: 1280px) { .ea-header { padding-top: 52px; max-width: calc(100vw - 28px); } .ea-header h1, .ea-header p, .ea-hint p, .ea-task-hint p { max-width: calc(100vw - 28px); white-space: normal; overflow-wrap: anywhere; } .edit-type-selector .wrap, .edit-type-selector [role="radiogroup"] { max-width: calc(100vw - 40px); } .edit-type-selector label { flex: 0 1 calc(50vw - 34px); max-width: calc(50vw - 34px); } } @media (max-width: 760px) { .ea-header { padding-top: 52px !important; } .edit-type-selector .wrap, .edit-type-selector [role="radiogroup"] { max-width: 100% !important; } .edit-type-selector label { flex: 1 1 100% !important; max-width: 100% !important; } } """ @spaces.GPU(duration=1) def _zerogpu_probe(): return "ready" print("Loading LTX-2.3 distilled diffusers pipeline...", flush=True) pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16) pipe.to("cuda") pipe.vae.enable_tiling() print("Loading Edit Anything standard LoRAs...", flush=True) for adapter_name, filename in ADAPTERS.values(): lora_path = hf_hub_download(EDIT_REPO, filename, token=HF_TOKEN) lora_state = load_file(lora_path) alpha_keys = [key for key in lora_state if key.endswith(".alpha")] if alpha_keys: print(f"Filtering {len(alpha_keys)} LoRA alpha tensors from {filename}.", flush=True) lora_state = {key: value for key, value in lora_state.items() if key not in alpha_keys} pipe.load_lora_weights(lora_state, adapter_name=adapter_name) del lora_state gc.collect() pipe.set_adapters(ADAPTERS[MODE_PROMPT][0], 1.0) print("Loading stage-2 spatial latent upsampler...", flush=True) _upsampler = LTX2LatentUpsamplerModel.from_pretrained( UPSAMPLER_REPO, subfolder="latent_upsampler", torch_dtype=torch.bfloat16, ) _upsampler.to("cuda") upsample_pipe = LTX2LatentUpsamplePipeline(vae=pipe.vae, latent_upsampler=_upsampler) print("Pipeline ready.", flush=True) def _src_fps(path, default=FPS): try: return float(iio.immeta(path, plugin="pyav").get("fps", default)) or default except Exception: return default def _probe_video(path): frames = load_video(path) if not frames: raise gr.Error("Could not read frames from the uploaded video.") return frames def _pick_resolution(first_frame, preset): width, height = RES_PRESETS[preset] if first_frame.height > first_frame.width: width, height = height, width return width, height def _load_frames(path, num_frames, width, height): frames = _probe_video(path) source_fps = _src_fps(path) out = [] for i in range(num_frames): idx = min(int(round(i / FPS * source_fps)), len(frames) - 1) frame = frames[idx].convert("RGB") out.append(ImageOps.fit(frame, (width, height), Image.LANCZOS)) return out def _prepare_first_frame(image, width, height): if image is None: raise gr.Error("Motion Transfer needs an externally edited first frame.") if not isinstance(image, Image.Image): image = Image.fromarray(np.asarray(image)) return ImageOps.fit(image.convert("RGB"), (width, height), Image.LANCZOS) def _compose_prompt(mode, edit_type, prompt, style_name): prompt = (prompt or "").strip() style_name = (style_name or "").strip() if mode == MODE_REF: raise gr.Error( "Ref V2V is not enabled in this diffusers Space. Its .module.safetensors " "sidecar installs custom AdaLN, role embedding, and ref-attention branches " "through BFSnodes; those branches are not standard diffusers LoRA adapters." ) if mode == MODE_PROMPT and edit_type == "Style": style = style_name or prompt if not style: raise gr.Error("Style mode needs a style name, for example 'Watercolor Painting'.") if style.lower().startswith("convert the video into"): return style return f"Convert the video into a {style} style." if not prompt: raise gr.Error("Enter an edit prompt.") return prompt def _duration(*args, **kwargs): preset = next((a for a in args if isinstance(a, str) and a in RES_PRESETS), "Fast (768x448)") num_frames = next((a for a in args if isinstance(a, int) and a in FRAME_CHOICES), 73) per_frame = 1.05 if "Quality" in str(preset) else 0.75 return int(15 + int(num_frames) * per_frame) def _export(video_np, audio, path): kwargs = {} if audio is not None: kwargs = { "audio": audio[0].float().cpu(), "audio_sample_rate": pipe.vocoder.config.output_sampling_rate, } encode_video(video_np, fps=FPS, output_path=path, **kwargs) def _set_adapter(mode, scale): adapter_name = ADAPTERS[mode][0] pipe.set_adapters(adapter_name, float(scale)) return adapter_name def _run_two_stage( prompt, reference_conditions, conditions, width, height, num_frames, seed, adapter_name, lora_scale, conditioning_attention_strength, ): pipe.set_adapters(adapter_name, float(lora_scale)) generator = torch.Generator(device="cuda").manual_seed(int(seed)) video_latent, audio_latent = pipe( prompt=prompt, negative_prompt="", reference_conditions=reference_conditions, conditions=conditions, reference_downscale_factor=1, conditioning_attention_strength=float(conditioning_attention_strength), width=width, height=height, num_frames=num_frames, frame_rate=FPS, num_inference_steps=NUM_STEPS, sigmas=DISTILLED_SIGMA_VALUES, guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0, generator=generator, output_type="latent", return_dict=False, ) up_latent = upsample_pipe(latents=video_latent, output_type="latent", return_dict=False)[0] pipe.disable_lora() try: video_out, audio_out = pipe( prompt=prompt, negative_prompt="", latents=up_latent, audio_latents=audio_latent, width=width * 2, height=height * 2, num_frames=num_frames, frame_rate=FPS, num_inference_steps=len(STAGE_2_DISTILLED_SIGMA_VALUES), sigmas=STAGE_2_DISTILLED_SIGMA_VALUES, noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0], guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0, generator=generator, output_type="np", return_dict=False, ) finally: pipe.set_adapters(adapter_name, float(lora_scale)) return video_out, audio_out @spaces.GPU(duration=_duration, size="xlarge") @torch.inference_mode() def edit_anything( mode, video, edited_first_frame, edit_type, prompt, style_name, preset, num_frames, seed, randomize_seed, lora_scale, guide_strength, source_attention, progress=gr.Progress(track_tqdm=True), ): if video is None: raise gr.Error("Upload a source video.") if mode not in ADAPTERS and mode != MODE_REF: raise gr.Error("Choose a supported edit mode.") final_prompt = _compose_prompt(mode, edit_type, prompt, style_name) if randomize_seed: seed = random.randint(0, MAX_SEED) seed = int(seed) num_frames = int(num_frames) progress(0.03, desc="Preparing source frames") first = _probe_video(video)[0].convert("RGB") width, height = _pick_resolution(first, preset) guide_frames = _load_frames(video, num_frames, width, height) reference_conditions = [ LTX2ReferenceCondition(frames=guide_frames, strength=float(guide_strength)) ] conditions = None edited_anchor = None if mode == MODE_MOTION: edited_anchor = _prepare_first_frame(edited_first_frame, width, height) conditions = [LTX2VideoCondition(frames=edited_anchor, index=0, strength=1.0)] adapter_name = _set_adapter(mode, lora_scale) started = time.perf_counter() progress(0.12, desc="Running LTX-2.3 stage 1") video_out, audio_out = _run_two_stage( prompt=final_prompt, reference_conditions=reference_conditions, conditions=conditions, width=width, height=height, num_frames=num_frames, seed=seed, adapter_name=adapter_name, lora_scale=lora_scale, conditioning_attention_strength=source_attention, ) progress(0.92, desc="Encoding output video") result = np.clip(video_out[0], 0, 1).astype(np.float32) if edited_anchor is not None and len(result) > 0: result[0] = ( np.array(edited_anchor.resize((width * 2, height * 2), Image.LANCZOS)).astype(np.float32) / 255.0 ) out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name _export(result, audio_out, out_path) elapsed = time.perf_counter() - started print( f"[METRIC] mode={mode!r} frames={num_frames} preset={preset!r} " f"seed={seed} elapsed_s={elapsed:.2f}", flush=True, ) details = ( f"Seed: {seed}\n" f"Prompt: {final_prompt}\n" f"Mode: {mode}\n" f"Elapsed seconds: {elapsed:.2f}" ) return out_path, seed, details def _mode_hint(mode): if mode == MODE_MOTION: return ( "Use a guide video plus one edited first frame. The frame anchors appearance; the video supplies motion." ) if mode == MODE_PROMPT: return "Upload a source video. Choose one focused edit task." return ( "Ref V2V is disabled in this diffusers build because it requires BFSnodes sidecar module injection." ) def _edit_type_hint(edit_type): if edit_type == "Add": return "Describe what to add and where it should appear." if edit_type == "Remove": return "Name what to remove. Short prompts work best." if edit_type == "Replace": return "Describe what changes and where." return "Enter a style name, e.g. Watercolor or Vintage Film." EDIT_PLACEHOLDERS = { "Add": "Add a cat beside the suitcase.", "Remove": "Remove the woman walking on the path.", "Replace": "Replace the statue with a man.", "Style": "", } def _mode_updates(mode): return _mode_hint(mode), gr.update(visible=(mode == MODE_MOTION)) def _edit_type_updates(edit_type): return ( _edit_type_hint(edit_type), gr.update( visible=(edit_type != "Style"), placeholder=EDIT_PLACEHOLDERS.get(edit_type, EDIT_PLACEHOLDERS["Replace"]), ), gr.update(visible=(edit_type == "Style")), ) def _run_prompt_example(mode, video, edit_type, prompt, style_name): return edit_anything( mode, video, UNUSED_FIRST_FRAME_EXAMPLE, edit_type, prompt, style_name, "Fast (768x448)", 49, 42, False, 1.0, 1.0, 1.0, ) def _run_motion_example(mode, video, edited_first_frame, prompt, style_name): return edit_anything( mode, video, edited_first_frame, "Replace", prompt, style_name, "Fast (768x448)", 49, 42, False, 1.0, 1.0, 1.0, ) with gr.Blocks( title="Edit Anything LTX-2.3", theme=APP_THEME, css=CUSTOM_CSS, ) as demo: gr.Markdown( "# Edit Anything LTX-2.3\n" "Prompt and first-frame video edits powered by Edit Anything LoRAs on LTX-2.3 Distilled.", elem_classes=["ea-header"], ) with gr.Row(elem_classes=["ea-main-row"]): with gr.Column(scale=1, min_width=0, elem_classes=["ea-input-column"]): gr.Markdown("## Inputs", elem_classes=["ea-column-title"]) mode_hint = gr.Markdown(_mode_hint(MODE_PROMPT), elem_classes=["ea-hint"]) mode = gr.Dropdown( [MODE_PROMPT, MODE_MOTION, MODE_REF], value=MODE_PROMPT, label="Workflow", info="Prompt for text edits. Motion uses a first frame.", ) video_in = gr.Video(label="Source video", height=260) with gr.Group(visible=False) as motion_anchor: edited_frame = gr.Image( label="Edited first frame", type="pil", image_mode="RGB", height=230, ) edit_type = gr.Radio( ["Add", "Remove", "Replace", "Style"], value="Replace", label="Edit task", elem_classes=["edit-type-selector"], ) edit_hint = gr.Markdown(_edit_type_hint("Replace"), elem_classes=["ea-task-hint"]) prompt = gr.Textbox( label="Edit prompt", lines=4, placeholder=EDIT_PLACEHOLDERS["Replace"], ) style_name = gr.Textbox( label="Style name", placeholder="Watercolor Painting", visible=False, ) with gr.Accordion("Settings", open=False): preset = gr.Dropdown(list(RES_PRESETS), value="Fast (768x448)", label="Resolution") num_frames = gr.Dropdown(FRAME_CHOICES, value=73, label="Frames at 24 fps") randomize_seed = gr.Checkbox(True, label="Randomize seed") seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed") lora_scale = gr.Slider(0.2, 1.4, value=1.0, step=0.05, label="LoRA scale") guide_strength = gr.Slider(0.2, 1.0, value=1.0, step=0.05, label="Guide video strength") source_attention = gr.Slider( 0.2, 1.0, value=1.0, step=0.05, label="Source/reference attention", ) run = gr.Button("Generate edited video", variant="primary", elem_classes=["ea-generate"]) with gr.Column(scale=1, min_width=0, elem_classes=["ea-output-column"]): gr.Markdown("## Output", elem_classes=["ea-column-title"]) video_out = gr.Video(label="Edited video", height=300) with gr.Accordion("Generation details", open=False): details = gr.Textbox(label="Run details", lines=5, show_label=False) with gr.Accordion("Examples", open=True, elem_classes=["ea-examples"]): with gr.Tabs(): with gr.Tab("Prompt"): gr.Examples( examples=PROMPT_EDIT_EXAMPLES, inputs=[mode, video_in, edit_type, prompt, style_name], outputs=[video_out, seed, details], fn=_run_prompt_example, cache_examples=True, cache_mode="lazy", examples_per_page=4, label="Prompt Edit examples", ) with gr.Tab("Motion"): gr.Examples( examples=MOTION_TRANSFER_EXAMPLES, inputs=[mode, video_in, edited_frame, prompt, style_name], outputs=[video_out, seed, details], fn=_run_motion_example, cache_examples=True, cache_mode="lazy", examples_per_page=1, label="Motion Transfer example", ) generate_inputs = [ mode, video_in, edited_frame, edit_type, prompt, style_name, preset, num_frames, seed, randomize_seed, lora_scale, guide_strength, source_attention, ] generate_outputs = [video_out, seed, details] mode.change(_mode_updates, inputs=mode, outputs=[mode_hint, motion_anchor]) edit_type.change(_edit_type_updates, inputs=edit_type, outputs=[edit_hint, prompt, style_name]) run.click( edit_anything, inputs=generate_inputs, outputs=generate_outputs, api_name="generate", ) if __name__ == "__main__": demo.launch(show_error=True)