Mike0021's picture
Adjust mobile header spacing
ff68a00 verified
Raw
History Blame Contribute Delete
23.7 kB
import os
os.environ.setdefault("HF_HOME", "/tmp/.cache/huggingface")
os.environ.setdefault("HF_MODULES_CACHE", "/tmp/hf_modules")
os.environ.setdefault("MPLCONFIGDIR", "/tmp/matplotlib")
os.environ.setdefault("GRADIO_SSR_MODE", "false")
os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")
os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
for _path in (
os.environ["HF_HOME"],
os.environ["HF_MODULES_CACHE"],
os.environ["MPLCONFIGDIR"],
):
os.makedirs(_path, exist_ok=True)
import random
import gc
import tempfile
import time
import gradio as gr
import imageio.v3 as iio
import numpy as np
import spaces
import torch
from huggingface_hub import hf_hub_download
from PIL import Image, ImageOps
from safetensors.torch import load_file
from diffusers import LTX2InContextPipeline, LTX2LatentUpsamplePipeline
from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition
from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition
from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES
from diffusers.utils import encode_video, load_video
BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers"
EDIT_REPO = "Alissonerdx/EditAnything"
MOTION_LORA = "edit_anything_30k_v0.1_motion_transfer_r128.safetensors"
PROMPT_LORA = "edit_anything_v1.1_r256.safetensors"
UPSAMPLER_REPO = "dg845/LTX-2.3-Spatial-Upsampler-Diffusers"
FPS = 24
NUM_STEPS = len(DISTILLED_SIGMA_VALUES)
MAX_SEED = np.iinfo(np.int32).max
HF_TOKEN = os.environ.get("HF_TOKEN")
MODE_MOTION = "Motion Transfer (v0.1)"
MODE_PROMPT = "Prompt Edit (v1.1)"
MODE_REF = "Ref V2V (experimental, not enabled)"
ADAPTERS = {
MODE_MOTION: ("motion_v01_r128", MOTION_LORA),
MODE_PROMPT: ("prompt_v11_r256", PROMPT_LORA),
}
RES_PRESETS = {
"Fast (768x448)": (768, 448),
"Quality (960x544)": (960, 544),
}
FRAME_CHOICES = [49, 73, 97, 121]
EXAMPLE_DIR = "examples"
UNUSED_FIRST_FRAME_EXAMPLE = f"{EXAMPLE_DIR}/motion_edited_first_frame.png"
PROMPT_EDIT_EXAMPLES = [
[
MODE_PROMPT,
f"{EXAMPLE_DIR}/prompt_add_source.mp4",
"Add",
"Add a golden retriever sitting on the grass beside the suitcase, to the right of the woman on the park bench.",
"",
],
[
MODE_PROMPT,
f"{EXAMPLE_DIR}/prompt_remove_source.mp4",
"Remove",
"Remove the woman walking on the path.",
"",
],
[
MODE_PROMPT,
f"{EXAMPLE_DIR}/prompt_replace_source.mp4",
"Replace",
"Replace the large red triangular sculpture in the background with a stone fountain spraying water in the plaza.",
"",
],
[
MODE_PROMPT,
f"{EXAMPLE_DIR}/prompt_style_source.mp4",
"Style",
"",
"Watercolor Painting",
],
]
MOTION_TRANSFER_EXAMPLES = [
[
MODE_MOTION,
f"{EXAMPLE_DIR}/motion_guide_source.mp4",
f"{EXAMPLE_DIR}/motion_edited_first_frame.png",
"Change the dancer into a copper-red-haired dancer wearing a loose white sweater and dark pants on the same rocky beach.",
"",
],
]
APP_THEME = gr.themes.Soft(
primary_hue="orange",
secondary_hue="sky",
neutral_hue="slate",
spacing_size="sm",
radius_size="sm",
text_size="sm",
font=gr.themes.GoogleFont("Inter"),
).set(
button_primary_background_fill="#f97316",
button_primary_background_fill_hover="#ea580c",
button_primary_text_color="#ffffff",
block_border_width="1px",
block_shadow="none",
)
CUSTOM_CSS = """
:root {
--ea-max-width: 1280px;
}
html,
body,
gradio-app {
overflow-x: hidden !important;
}
.gradio-container,
.gradio-container * {
box-sizing: border-box;
}
.gradio-container {
width: min(var(--ea-max-width), calc(100vw - 32px)) !important;
max-width: var(--ea-max-width) !important;
margin: 0 auto !important;
color: #111827;
min-width: 0 !important;
}
.ea-main-row {
align-items: flex-start !important;
gap: 0.75rem !important;
width: 100% !important;
min-width: 0 !important;
}
.ea-main-row > *,
.ea-input-column,
.ea-output-column {
min-width: 0 !important;
max-width: 100% !important;
}
.ea-header {
margin: 0 0 14px;
}
.ea-header h1 {
font-size: clamp(1.45rem, 2vw, 2rem);
line-height: 1.12;
margin-bottom: 0.35rem;
letter-spacing: 0;
}
.ea-header p {
color: #64748b;
margin: 0;
}
.ea-column-title h2,
.ea-column-title h3 {
font-size: 0.95rem;
line-height: 1.25;
margin: 0 0 0.25rem;
}
.ea-hint p,
.ea-task-hint p {
color: #64748b;
font-size: 0.9rem;
margin: 0;
}
.ea-output-column {
position: sticky;
top: 12px;
align-self: start;
}
.ea-generate button {
min-height: 48px;
font-weight: 700;
letter-spacing: 0;
}
.edit-type-selector .wrap {
display: flex !important;
gap: 0.45rem;
flex-wrap: wrap !important;
}
.edit-type-selector [role="radiogroup"] {
display: flex !important;
flex-wrap: wrap !important;
gap: 0.45rem !important;
}
.edit-type-selector label {
flex: 1 1 120px;
min-width: 0;
border-radius: 8px !important;
border: 1px solid #cbd5e1 !important;
padding: 0.5rem 0.72rem !important;
background: #ffffff !important;
}
.edit-type-selector label:has(input:checked) {
border-color: #f97316 !important;
background: #fff7ed !important;
color: #9a3412 !important;
font-weight: 700;
}
.ea-examples table {
font-size: 0.82rem;
}
.ea-examples .table-wrap {
max-height: 320px;
overflow: auto;
max-width: 100%;
}
footer {
display: none !important;
}
@media (max-width: 1000px) {
.gradio-container {
padding-top: 52px !important;
padding-left: 14px !important;
padding-right: 14px !important;
overflow-x: hidden !important;
}
.ea-output-column {
position: static;
}
}
@media (max-width: 760px) {
.gradio-container {
width: 100% !important;
max-width: 100vw !important;
padding: 56px 16px 24px !important;
}
.ea-header {
padding-top: 0 !important;
max-width: 100% !important;
}
.ea-main-row {
display: flex !important;
flex-direction: column !important;
gap: 0.85rem !important;
}
.ea-main-row > *,
.ea-input-column,
.ea-output-column,
.gradio-container .block,
.gradio-container .form,
.gradio-container .panel,
.gradio-container .tabs,
.gradio-container .tabitem {
width: 100% !important;
max-width: 100% !important;
min-width: 0 !important;
flex: 1 1 auto !important;
}
.ea-output-column {
margin-top: 0.25rem;
}
.edit-type-selector .wrap,
.edit-type-selector [role="radiogroup"] {
width: 100% !important;
max-width: 100% !important;
}
.edit-type-selector label {
flex: 1 1 100% !important;
max-width: 100% !important;
min-height: 46px;
display: flex !important;
align-items: center;
}
.ea-generate button {
min-height: 52px;
}
.ea-examples .table-wrap {
overflow-x: auto;
}
.ea-examples table {
min-width: 560px;
}
}
@media (max-width: 1280px) {
.ea-header {
padding-top: 52px;
max-width: calc(100vw - 28px);
}
.ea-header h1,
.ea-header p,
.ea-hint p,
.ea-task-hint p {
max-width: calc(100vw - 28px);
white-space: normal;
overflow-wrap: anywhere;
}
.edit-type-selector .wrap,
.edit-type-selector [role="radiogroup"] {
max-width: calc(100vw - 40px);
}
.edit-type-selector label {
flex: 0 1 calc(50vw - 34px);
max-width: calc(50vw - 34px);
}
}
@media (max-width: 760px) {
.ea-header {
padding-top: 52px !important;
}
.edit-type-selector .wrap,
.edit-type-selector [role="radiogroup"] {
max-width: 100% !important;
}
.edit-type-selector label {
flex: 1 1 100% !important;
max-width: 100% !important;
}
}
"""
@spaces.GPU(duration=1)
def _zerogpu_probe():
return "ready"
print("Loading LTX-2.3 distilled diffusers pipeline...", flush=True)
pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16)
pipe.to("cuda")
pipe.vae.enable_tiling()
print("Loading Edit Anything standard LoRAs...", flush=True)
for adapter_name, filename in ADAPTERS.values():
lora_path = hf_hub_download(EDIT_REPO, filename, token=HF_TOKEN)
lora_state = load_file(lora_path)
alpha_keys = [key for key in lora_state if key.endswith(".alpha")]
if alpha_keys:
print(f"Filtering {len(alpha_keys)} LoRA alpha tensors from {filename}.", flush=True)
lora_state = {key: value for key, value in lora_state.items() if key not in alpha_keys}
pipe.load_lora_weights(lora_state, adapter_name=adapter_name)
del lora_state
gc.collect()
pipe.set_adapters(ADAPTERS[MODE_PROMPT][0], 1.0)
print("Loading stage-2 spatial latent upsampler...", flush=True)
_upsampler = LTX2LatentUpsamplerModel.from_pretrained(
UPSAMPLER_REPO,
subfolder="latent_upsampler",
torch_dtype=torch.bfloat16,
)
_upsampler.to("cuda")
upsample_pipe = LTX2LatentUpsamplePipeline(vae=pipe.vae, latent_upsampler=_upsampler)
print("Pipeline ready.", flush=True)
def _src_fps(path, default=FPS):
try:
return float(iio.immeta(path, plugin="pyav").get("fps", default)) or default
except Exception:
return default
def _probe_video(path):
frames = load_video(path)
if not frames:
raise gr.Error("Could not read frames from the uploaded video.")
return frames
def _pick_resolution(first_frame, preset):
width, height = RES_PRESETS[preset]
if first_frame.height > first_frame.width:
width, height = height, width
return width, height
def _load_frames(path, num_frames, width, height):
frames = _probe_video(path)
source_fps = _src_fps(path)
out = []
for i in range(num_frames):
idx = min(int(round(i / FPS * source_fps)), len(frames) - 1)
frame = frames[idx].convert("RGB")
out.append(ImageOps.fit(frame, (width, height), Image.LANCZOS))
return out
def _prepare_first_frame(image, width, height):
if image is None:
raise gr.Error("Motion Transfer needs an externally edited first frame.")
if not isinstance(image, Image.Image):
image = Image.fromarray(np.asarray(image))
return ImageOps.fit(image.convert("RGB"), (width, height), Image.LANCZOS)
def _compose_prompt(mode, edit_type, prompt, style_name):
prompt = (prompt or "").strip()
style_name = (style_name or "").strip()
if mode == MODE_REF:
raise gr.Error(
"Ref V2V is not enabled in this diffusers Space. Its .module.safetensors "
"sidecar installs custom AdaLN, role embedding, and ref-attention branches "
"through BFSnodes; those branches are not standard diffusers LoRA adapters."
)
if mode == MODE_PROMPT and edit_type == "Style":
style = style_name or prompt
if not style:
raise gr.Error("Style mode needs a style name, for example 'Watercolor Painting'.")
if style.lower().startswith("convert the video into"):
return style
return f"Convert the video into a {style} style."
if not prompt:
raise gr.Error("Enter an edit prompt.")
return prompt
def _duration(*args, **kwargs):
preset = next((a for a in args if isinstance(a, str) and a in RES_PRESETS), "Fast (768x448)")
num_frames = next((a for a in args if isinstance(a, int) and a in FRAME_CHOICES), 73)
per_frame = 1.05 if "Quality" in str(preset) else 0.75
return int(15 + int(num_frames) * per_frame)
def _export(video_np, audio, path):
kwargs = {}
if audio is not None:
kwargs = {
"audio": audio[0].float().cpu(),
"audio_sample_rate": pipe.vocoder.config.output_sampling_rate,
}
encode_video(video_np, fps=FPS, output_path=path, **kwargs)
def _set_adapter(mode, scale):
adapter_name = ADAPTERS[mode][0]
pipe.set_adapters(adapter_name, float(scale))
return adapter_name
def _run_two_stage(
prompt,
reference_conditions,
conditions,
width,
height,
num_frames,
seed,
adapter_name,
lora_scale,
conditioning_attention_strength,
):
pipe.set_adapters(adapter_name, float(lora_scale))
generator = torch.Generator(device="cuda").manual_seed(int(seed))
video_latent, audio_latent = pipe(
prompt=prompt,
negative_prompt="",
reference_conditions=reference_conditions,
conditions=conditions,
reference_downscale_factor=1,
conditioning_attention_strength=float(conditioning_attention_strength),
width=width,
height=height,
num_frames=num_frames,
frame_rate=FPS,
num_inference_steps=NUM_STEPS,
sigmas=DISTILLED_SIGMA_VALUES,
guidance_scale=1.0,
stg_scale=0.0,
audio_guidance_scale=1.0,
audio_stg_scale=0.0,
generator=generator,
output_type="latent",
return_dict=False,
)
up_latent = upsample_pipe(latents=video_latent, output_type="latent", return_dict=False)[0]
pipe.disable_lora()
try:
video_out, audio_out = pipe(
prompt=prompt,
negative_prompt="",
latents=up_latent,
audio_latents=audio_latent,
width=width * 2,
height=height * 2,
num_frames=num_frames,
frame_rate=FPS,
num_inference_steps=len(STAGE_2_DISTILLED_SIGMA_VALUES),
sigmas=STAGE_2_DISTILLED_SIGMA_VALUES,
noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0],
guidance_scale=1.0,
stg_scale=0.0,
audio_guidance_scale=1.0,
audio_stg_scale=0.0,
generator=generator,
output_type="np",
return_dict=False,
)
finally:
pipe.set_adapters(adapter_name, float(lora_scale))
return video_out, audio_out
@spaces.GPU(duration=_duration, size="xlarge")
@torch.inference_mode()
def edit_anything(
mode,
video,
edited_first_frame,
edit_type,
prompt,
style_name,
preset,
num_frames,
seed,
randomize_seed,
lora_scale,
guide_strength,
source_attention,
progress=gr.Progress(track_tqdm=True),
):
if video is None:
raise gr.Error("Upload a source video.")
if mode not in ADAPTERS and mode != MODE_REF:
raise gr.Error("Choose a supported edit mode.")
final_prompt = _compose_prompt(mode, edit_type, prompt, style_name)
if randomize_seed:
seed = random.randint(0, MAX_SEED)
seed = int(seed)
num_frames = int(num_frames)
progress(0.03, desc="Preparing source frames")
first = _probe_video(video)[0].convert("RGB")
width, height = _pick_resolution(first, preset)
guide_frames = _load_frames(video, num_frames, width, height)
reference_conditions = [
LTX2ReferenceCondition(frames=guide_frames, strength=float(guide_strength))
]
conditions = None
edited_anchor = None
if mode == MODE_MOTION:
edited_anchor = _prepare_first_frame(edited_first_frame, width, height)
conditions = [LTX2VideoCondition(frames=edited_anchor, index=0, strength=1.0)]
adapter_name = _set_adapter(mode, lora_scale)
started = time.perf_counter()
progress(0.12, desc="Running LTX-2.3 stage 1")
video_out, audio_out = _run_two_stage(
prompt=final_prompt,
reference_conditions=reference_conditions,
conditions=conditions,
width=width,
height=height,
num_frames=num_frames,
seed=seed,
adapter_name=adapter_name,
lora_scale=lora_scale,
conditioning_attention_strength=source_attention,
)
progress(0.92, desc="Encoding output video")
result = np.clip(video_out[0], 0, 1).astype(np.float32)
if edited_anchor is not None and len(result) > 0:
result[0] = (
np.array(edited_anchor.resize((width * 2, height * 2), Image.LANCZOS)).astype(np.float32) / 255.0
)
out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
_export(result, audio_out, out_path)
elapsed = time.perf_counter() - started
print(
f"[METRIC] mode={mode!r} frames={num_frames} preset={preset!r} "
f"seed={seed} elapsed_s={elapsed:.2f}",
flush=True,
)
details = (
f"Seed: {seed}\n"
f"Prompt: {final_prompt}\n"
f"Mode: {mode}\n"
f"Elapsed seconds: {elapsed:.2f}"
)
return out_path, seed, details
def _mode_hint(mode):
if mode == MODE_MOTION:
return (
"Use a guide video plus one edited first frame. The frame anchors appearance; the video supplies motion."
)
if mode == MODE_PROMPT:
return "Upload a source video. Choose one focused edit task."
return (
"Ref V2V is disabled in this diffusers build because it requires BFSnodes sidecar module injection."
)
def _edit_type_hint(edit_type):
if edit_type == "Add":
return "Describe what to add and where it should appear."
if edit_type == "Remove":
return "Name what to remove. Short prompts work best."
if edit_type == "Replace":
return "Describe what changes and where."
return "Enter a style name, e.g. Watercolor or Vintage Film."
EDIT_PLACEHOLDERS = {
"Add": "Add a cat beside the suitcase.",
"Remove": "Remove the woman walking on the path.",
"Replace": "Replace the statue with a man.",
"Style": "",
}
def _mode_updates(mode):
return _mode_hint(mode), gr.update(visible=(mode == MODE_MOTION))
def _edit_type_updates(edit_type):
return (
_edit_type_hint(edit_type),
gr.update(
visible=(edit_type != "Style"),
placeholder=EDIT_PLACEHOLDERS.get(edit_type, EDIT_PLACEHOLDERS["Replace"]),
),
gr.update(visible=(edit_type == "Style")),
)
def _run_prompt_example(mode, video, edit_type, prompt, style_name):
return edit_anything(
mode,
video,
UNUSED_FIRST_FRAME_EXAMPLE,
edit_type,
prompt,
style_name,
"Fast (768x448)",
49,
42,
False,
1.0,
1.0,
1.0,
)
def _run_motion_example(mode, video, edited_first_frame, prompt, style_name):
return edit_anything(
mode,
video,
edited_first_frame,
"Replace",
prompt,
style_name,
"Fast (768x448)",
49,
42,
False,
1.0,
1.0,
1.0,
)
with gr.Blocks(
title="Edit Anything LTX-2.3",
theme=APP_THEME,
css=CUSTOM_CSS,
) as demo:
gr.Markdown(
"# Edit Anything LTX-2.3\n"
"Prompt and first-frame video edits powered by Edit Anything LoRAs on LTX-2.3 Distilled.",
elem_classes=["ea-header"],
)
with gr.Row(elem_classes=["ea-main-row"]):
with gr.Column(scale=1, min_width=0, elem_classes=["ea-input-column"]):
gr.Markdown("## Inputs", elem_classes=["ea-column-title"])
mode_hint = gr.Markdown(_mode_hint(MODE_PROMPT), elem_classes=["ea-hint"])
mode = gr.Dropdown(
[MODE_PROMPT, MODE_MOTION, MODE_REF],
value=MODE_PROMPT,
label="Workflow",
info="Prompt for text edits. Motion uses a first frame.",
)
video_in = gr.Video(label="Source video", height=260)
with gr.Group(visible=False) as motion_anchor:
edited_frame = gr.Image(
label="Edited first frame",
type="pil",
image_mode="RGB",
height=230,
)
edit_type = gr.Radio(
["Add", "Remove", "Replace", "Style"],
value="Replace",
label="Edit task",
elem_classes=["edit-type-selector"],
)
edit_hint = gr.Markdown(_edit_type_hint("Replace"), elem_classes=["ea-task-hint"])
prompt = gr.Textbox(
label="Edit prompt",
lines=4,
placeholder=EDIT_PLACEHOLDERS["Replace"],
)
style_name = gr.Textbox(
label="Style name",
placeholder="Watercolor Painting",
visible=False,
)
with gr.Accordion("Settings", open=False):
preset = gr.Dropdown(list(RES_PRESETS), value="Fast (768x448)", label="Resolution")
num_frames = gr.Dropdown(FRAME_CHOICES, value=73, label="Frames at 24 fps")
randomize_seed = gr.Checkbox(True, label="Randomize seed")
seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed")
lora_scale = gr.Slider(0.2, 1.4, value=1.0, step=0.05, label="LoRA scale")
guide_strength = gr.Slider(0.2, 1.0, value=1.0, step=0.05, label="Guide video strength")
source_attention = gr.Slider(
0.2,
1.0,
value=1.0,
step=0.05,
label="Source/reference attention",
)
run = gr.Button("Generate edited video", variant="primary", elem_classes=["ea-generate"])
with gr.Column(scale=1, min_width=0, elem_classes=["ea-output-column"]):
gr.Markdown("## Output", elem_classes=["ea-column-title"])
video_out = gr.Video(label="Edited video", height=300)
with gr.Accordion("Generation details", open=False):
details = gr.Textbox(label="Run details", lines=5, show_label=False)
with gr.Accordion("Examples", open=True, elem_classes=["ea-examples"]):
with gr.Tabs():
with gr.Tab("Prompt"):
gr.Examples(
examples=PROMPT_EDIT_EXAMPLES,
inputs=[mode, video_in, edit_type, prompt, style_name],
outputs=[video_out, seed, details],
fn=_run_prompt_example,
cache_examples=True,
cache_mode="lazy",
examples_per_page=4,
label="Prompt Edit examples",
)
with gr.Tab("Motion"):
gr.Examples(
examples=MOTION_TRANSFER_EXAMPLES,
inputs=[mode, video_in, edited_frame, prompt, style_name],
outputs=[video_out, seed, details],
fn=_run_motion_example,
cache_examples=True,
cache_mode="lazy",
examples_per_page=1,
label="Motion Transfer example",
)
generate_inputs = [
mode,
video_in,
edited_frame,
edit_type,
prompt,
style_name,
preset,
num_frames,
seed,
randomize_seed,
lora_scale,
guide_strength,
source_attention,
]
generate_outputs = [video_out, seed, details]
mode.change(_mode_updates, inputs=mode, outputs=[mode_hint, motion_anchor])
edit_type.change(_edit_type_updates, inputs=edit_type, outputs=[edit_hint, prompt, style_name])
run.click(
edit_anything,
inputs=generate_inputs,
outputs=generate_outputs,
api_name="generate",
)
if __name__ == "__main__":
demo.launch(show_error=True)