Fabrice-TIERCELIN commited on
Commit
9910db6
·
verified ·
1 Parent(s): 4bdf193

Global before

Browse files
Files changed (1) hide show
  1. app.py +622 -622
app.py CHANGED
@@ -1,623 +1,623 @@
1
- import gradio as gr
2
- import torch
3
- import spaces
4
- import numpy as np
5
- import random
6
- import os
7
- import yaml
8
- from pathlib import Path
9
- import imageio
10
- import tempfile
11
- from PIL import Image
12
- from huggingface_hub import hf_hub_download
13
- import shutil
14
-
15
- from inference import (
16
- create_ltx_video_pipeline,
17
- create_latent_upsampler,
18
- load_image_to_tensor_with_resize_and_crop,
19
- seed_everething,
20
- get_device,
21
- calculate_padding,
22
- load_media_file
23
- )
24
- from ltx_video.pipelines.pipeline_ltx_video import ConditioningItem, LTXMultiScalePipeline, LTXVideoPipeline
25
- from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
26
-
27
- image_i2v_debug_value = None
28
- i2v_prompt_debug_value = None
29
- height_input_debug_value = None
30
- width_input_debug_value = None
31
- duration_input_debug_value = None
32
- config_file_path = "configs/ltxv-13b-0.9.7-distilled.yaml"
33
- with open(config_file_path, "r") as file:
34
- PIPELINE_CONFIG_YAML = yaml.safe_load(file)
35
-
36
- LTX_REPO = "Lightricks/LTX-Video"
37
- MAX_IMAGE_SIZE = PIPELINE_CONFIG_YAML.get("max_resolution", 1280)
38
- MAX_NUM_FRAMES = 257
39
-
40
- FPS = 30.0
41
-
42
- # --- Global variables for loaded models ---
43
- pipeline_instance = None
44
- latent_upsampler_instance = None
45
- models_dir = "downloaded_models_gradio_cpu_init"
46
- Path(models_dir).mkdir(parents=True, exist_ok=True)
47
-
48
- print("Downloading models (if not present)...")
49
- distilled_model_actual_path = hf_hub_download(
50
- repo_id=LTX_REPO,
51
- filename=PIPELINE_CONFIG_YAML["checkpoint_path"],
52
- local_dir=models_dir,
53
- local_dir_use_symlinks=False
54
- )
55
- PIPELINE_CONFIG_YAML["checkpoint_path"] = distilled_model_actual_path
56
- print(f"Distilled model path: {distilled_model_actual_path}")
57
-
58
- SPATIAL_UPSCALER_FILENAME = PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"]
59
- spatial_upscaler_actual_path = hf_hub_download(
60
- repo_id=LTX_REPO,
61
- filename=SPATIAL_UPSCALER_FILENAME,
62
- local_dir=models_dir,
63
- local_dir_use_symlinks=False
64
- )
65
- PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"] = spatial_upscaler_actual_path
66
- print(f"Spatial upscaler model path: {spatial_upscaler_actual_path}")
67
-
68
- print("Creating LTX Video pipeline on CPU...")
69
- pipeline_instance = create_ltx_video_pipeline(
70
- ckpt_path=PIPELINE_CONFIG_YAML["checkpoint_path"],
71
- precision=PIPELINE_CONFIG_YAML["precision"],
72
- text_encoder_model_name_or_path=PIPELINE_CONFIG_YAML["text_encoder_model_name_or_path"],
73
- sampler=PIPELINE_CONFIG_YAML["sampler"],
74
- device="cpu",
75
- enhance_prompt=False,
76
- prompt_enhancer_image_caption_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_image_caption_model_name_or_path"],
77
- prompt_enhancer_llm_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_llm_model_name_or_path"],
78
- )
79
- print("LTX Video pipeline created on CPU.")
80
-
81
- if PIPELINE_CONFIG_YAML.get("spatial_upscaler_model_path"):
82
- print("Creating latent upsampler on CPU...")
83
- latent_upsampler_instance = create_latent_upsampler(
84
- PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"],
85
- device="cpu"
86
- )
87
- print("Latent upsampler created on CPU.")
88
-
89
- target_inference_device = "cuda"
90
- print(f"Target inference device: {target_inference_device}")
91
- pipeline_instance.to(target_inference_device)
92
- if latent_upsampler_instance:
93
- latent_upsampler_instance.to(target_inference_device)
94
-
95
-
96
- # --- Helper function for dimension calculation ---
97
- MIN_DIM_SLIDER = 256 # As defined in the sliders minimum attribute
98
- TARGET_FIXED_SIDE = 768 # Desired fixed side length as per requirement
99
-
100
- def calculate_new_dimensions(orig_w, orig_h):
101
- """
102
- Calculates new dimensions for height and width sliders based on original media dimensions.
103
- Ensures one side is TARGET_FIXED_SIDE, the other is scaled proportionally,
104
- both are multiples of 32, and within [MIN_DIM_SLIDER, MAX_IMAGE_SIZE].
105
- """
106
- if orig_w == 0 or orig_h == 0:
107
- # Default to TARGET_FIXED_SIDE square if original dimensions are invalid
108
- return int(TARGET_FIXED_SIDE), int(TARGET_FIXED_SIDE)
109
-
110
- if orig_w >= orig_h: # Landscape or square
111
- new_h = TARGET_FIXED_SIDE
112
- aspect_ratio = orig_w / orig_h
113
- new_w_ideal = new_h * aspect_ratio
114
-
115
- # Round to nearest multiple of 32
116
- new_w = round(new_w_ideal / 32) * 32
117
-
118
- # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
119
- new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
120
- # Ensure new_h is also clamped (TARGET_FIXED_SIDE should be within these bounds if configured correctly)
121
- new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE))
122
- else: # Portrait
123
- new_w = TARGET_FIXED_SIDE
124
- aspect_ratio = orig_h / orig_w # Use H/W ratio for portrait scaling
125
- new_h_ideal = new_w * aspect_ratio
126
-
127
- # Round to nearest multiple of 32
128
- new_h = round(new_h_ideal / 32) * 32
129
-
130
- # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
131
- new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE))
132
- # Ensure new_w is also clamped
133
- new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
134
-
135
- return int(new_h), int(new_w)
136
-
137
- def get_duration(prompt, negative_prompt, input_image_filepath, input_video_filepath,
138
- height_ui, width_ui, mode,
139
- duration_ui, # Removed ui_steps
140
- ui_frames_to_use,
141
- seed_ui, randomize_seed, ui_guidance_scale, improve_texture_flag,
142
- progress):
143
- if duration_ui > 7:
144
- return 75
145
- else:
146
- return 60
147
-
148
- @spaces.GPU(duration=get_duration)
149
- def generate(prompt, negative_prompt, input_image_filepath, input_video_filepath,
150
- height_ui, width_ui, mode,
151
- duration_ui,
152
- ui_frames_to_use,
153
- seed_ui, randomize_seed, ui_guidance_scale, improve_texture_flag,
154
- progress=gr.Progress(track_tqdm=True)):
155
-
156
- if i2v_prompt_debug_value is not None:
157
- prompt = i2v_prompt_debug_value
158
- global i2v_prompt_debug_value
159
- i2v_prompt_debug_value = None
160
-
161
- if image_i2v_debug_value is not None:
162
- input_image_filepath = image_i2v_debug_value
163
- global image_i2v_debug_value
164
- image_i2v_debug_value = None
165
-
166
- if height_input_debug_value is not None:
167
- height_ui = height_input_debug_value
168
- global height_input_debug_value
169
- height_input_debug_value = None
170
-
171
- if width_input_debug_value is not None:
172
- width_ui = width_input_debug_value
173
- global width_input_debug_value
174
- width_input_debug_value = None
175
-
176
- if duration_input_debug_value is not None:
177
- duration_ui = duration_input_debug_value
178
- global duration_input_debug_value
179
- duration_input_debug_value = None
180
-
181
- if randomize_seed:
182
- seed_ui = random.randint(0, 2**32 - 1)
183
- seed_everething(int(seed_ui))
184
-
185
- target_frames_ideal = duration_ui * FPS
186
- target_frames_rounded = round(target_frames_ideal)
187
- if target_frames_rounded < 1:
188
- target_frames_rounded = 1
189
-
190
- n_val = round((float(target_frames_rounded) - 1.0) / 8.0)
191
- actual_num_frames = int(n_val * 8 + 1)
192
-
193
- actual_num_frames = max(9, actual_num_frames)
194
- actual_num_frames = min(MAX_NUM_FRAMES, actual_num_frames)
195
-
196
- actual_height = int(height_ui)
197
- actual_width = int(width_ui)
198
-
199
- height_padded = ((actual_height - 1) // 32 + 1) * 32
200
- width_padded = ((actual_width - 1) // 32 + 1) * 32
201
- num_frames_padded = ((actual_num_frames - 2) // 8 + 1) * 8 + 1
202
- if num_frames_padded != actual_num_frames:
203
- print(f"Warning: actual_num_frames ({actual_num_frames}) and num_frames_padded ({num_frames_padded}) differ. Using num_frames_padded for pipeline.")
204
-
205
- padding_values = calculate_padding(actual_height, actual_width, height_padded, width_padded)
206
-
207
- call_kwargs = {
208
- "prompt": prompt,
209
- "negative_prompt": negative_prompt,
210
- "height": height_padded,
211
- "width": width_padded,
212
- "num_frames": num_frames_padded,
213
- "frame_rate": int(FPS),
214
- "generator": torch.Generator(device=target_inference_device).manual_seed(int(seed_ui)),
215
- "output_type": "pt",
216
- "conditioning_items": None,
217
- "media_items": None,
218
- "decode_timestep": PIPELINE_CONFIG_YAML["decode_timestep"],
219
- "decode_noise_scale": PIPELINE_CONFIG_YAML["decode_noise_scale"],
220
- "stochastic_sampling": PIPELINE_CONFIG_YAML["stochastic_sampling"],
221
- "image_cond_noise_scale": 0.15,
222
- "is_video": True,
223
- "vae_per_channel_normalize": True,
224
- "mixed_precision": (PIPELINE_CONFIG_YAML["precision"] == "mixed_precision"),
225
- "offload_to_cpu": False,
226
- "enhance_prompt": False,
227
- }
228
-
229
- stg_mode_str = PIPELINE_CONFIG_YAML.get("stg_mode", "attention_values")
230
- if stg_mode_str.lower() in ["stg_av", "attention_values"]:
231
- call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionValues
232
- elif stg_mode_str.lower() in ["stg_as", "attention_skip"]:
233
- call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionSkip
234
- elif stg_mode_str.lower() in ["stg_r", "residual"]:
235
- call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.Residual
236
- elif stg_mode_str.lower() in ["stg_t", "transformer_block"]:
237
- call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.TransformerBlock
238
- else:
239
- raise ValueError(f"Invalid stg_mode: {stg_mode_str}")
240
-
241
- if mode == "image-to-video" and input_image_filepath:
242
- try:
243
- media_tensor = load_image_to_tensor_with_resize_and_crop(
244
- input_image_filepath, actual_height, actual_width
245
- )
246
- media_tensor = torch.nn.functional.pad(media_tensor, padding_values)
247
- call_kwargs["conditioning_items"] = [ConditioningItem(media_tensor.to(target_inference_device), 0, 1.0)]
248
- except Exception as e:
249
- print(f"Error loading image {input_image_filepath}: {e}")
250
- raise gr.Error(f"Could not load image: {e}")
251
- elif mode == "video-to-video" and input_video_filepath:
252
- try:
253
- call_kwargs["media_items"] = load_media_file(
254
- media_path=input_video_filepath,
255
- height=actual_height,
256
- width=actual_width,
257
- max_frames=int(ui_frames_to_use),
258
- padding=padding_values
259
- ).to(target_inference_device)
260
- except Exception as e:
261
- print(f"Error loading video {input_video_filepath}: {e}")
262
- raise gr.Error(f"Could not load video: {e}")
263
-
264
- print(f"Moving models to {target_inference_device} for inference (if not already there)...")
265
-
266
- active_latent_upsampler = None
267
- if improve_texture_flag and latent_upsampler_instance:
268
- active_latent_upsampler = latent_upsampler_instance
269
-
270
- result_images_tensor = None
271
- if improve_texture_flag:
272
- if not active_latent_upsampler:
273
- raise gr.Error("Spatial upscaler model not loaded or improve_texture not selected, cannot use multi-scale.")
274
-
275
- multi_scale_pipeline_obj = LTXMultiScalePipeline(pipeline_instance, active_latent_upsampler)
276
-
277
- first_pass_args = PIPELINE_CONFIG_YAML.get("first_pass", {}).copy()
278
- first_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
279
- # num_inference_steps will be derived from len(timesteps) in the pipeline
280
- first_pass_args.pop("num_inference_steps", None)
281
-
282
-
283
- second_pass_args = PIPELINE_CONFIG_YAML.get("second_pass", {}).copy()
284
- second_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
285
- # num_inference_steps will be derived from len(timesteps) in the pipeline
286
- second_pass_args.pop("num_inference_steps", None)
287
-
288
- multi_scale_call_kwargs = call_kwargs.copy()
289
- multi_scale_call_kwargs.update({
290
- "downscale_factor": PIPELINE_CONFIG_YAML["downscale_factor"],
291
- "first_pass": first_pass_args,
292
- "second_pass": second_pass_args,
293
- })
294
-
295
- print(f"Calling multi-scale pipeline (eff. HxW: {actual_height}x{actual_width}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
296
- result_images_tensor = multi_scale_pipeline_obj(**multi_scale_call_kwargs).images
297
- else:
298
- single_pass_call_kwargs = call_kwargs.copy()
299
- first_pass_config_from_yaml = PIPELINE_CONFIG_YAML.get("first_pass", {})
300
-
301
- single_pass_call_kwargs["timesteps"] = first_pass_config_from_yaml.get("timesteps")
302
- single_pass_call_kwargs["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
303
- single_pass_call_kwargs["stg_scale"] = first_pass_config_from_yaml.get("stg_scale")
304
- single_pass_call_kwargs["rescaling_scale"] = first_pass_config_from_yaml.get("rescaling_scale")
305
- single_pass_call_kwargs["skip_block_list"] = first_pass_config_from_yaml.get("skip_block_list")
306
-
307
- # Remove keys that might conflict or are not used in single pass / handled by above
308
- single_pass_call_kwargs.pop("num_inference_steps", None)
309
- single_pass_call_kwargs.pop("first_pass", None)
310
- single_pass_call_kwargs.pop("second_pass", None)
311
- single_pass_call_kwargs.pop("downscale_factor", None)
312
-
313
- print(f"Calling base pipeline (padded HxW: {height_padded}x{width_padded}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
314
- result_images_tensor = pipeline_instance(**single_pass_call_kwargs).images
315
-
316
- if result_images_tensor is None:
317
- raise gr.Error("Generation failed.")
318
-
319
- pad_left, pad_right, pad_top, pad_bottom = padding_values
320
- slice_h_end = -pad_bottom if pad_bottom > 0 else None
321
- slice_w_end = -pad_right if pad_right > 0 else None
322
-
323
- result_images_tensor = result_images_tensor[
324
- :, :, :actual_num_frames, pad_top:slice_h_end, pad_left:slice_w_end
325
- ]
326
-
327
- video_np = result_images_tensor[0].permute(1, 2, 3, 0).cpu().float().numpy()
328
-
329
- video_np = np.clip(video_np, 0, 1)
330
- video_np = (video_np * 255).astype(np.uint8)
331
-
332
- temp_dir = tempfile.mkdtemp()
333
- timestamp = random.randint(10000,99999)
334
- output_video_path = os.path.join(temp_dir, f"output_{timestamp}.mp4")
335
-
336
- try:
337
- with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], macro_block_size=1) as video_writer:
338
- for frame_idx in range(video_np.shape[0]):
339
- progress(frame_idx / video_np.shape[0], desc="Saving video")
340
- video_writer.append_data(video_np[frame_idx])
341
- except Exception as e:
342
- print(f"Error saving video with macro_block_size=1: {e}")
343
- try:
344
- with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], format='FFMPEG', codec='libx264', quality=8) as video_writer:
345
- for frame_idx in range(video_np.shape[0]):
346
- progress(frame_idx / video_np.shape[0], desc="Saving video (fallback ffmpeg)")
347
- video_writer.append_data(video_np[frame_idx])
348
- except Exception as e2:
349
- print(f"Fallback video saving error: {e2}")
350
- raise gr.Error(f"Failed to save video: {e2}")
351
-
352
- return output_video_path, seed_ui
353
-
354
- def update_task_image():
355
- return "image-to-video"
356
-
357
- def update_task_text():
358
- return "text-to-video"
359
-
360
- def update_task_video():
361
- return "video-to-video"
362
-
363
- # --- Gradio UI Definition ---
364
- css="""
365
- #col-container {
366
- margin: 0 auto;
367
- max-width: 900px;
368
- }
369
- """
370
-
371
- with gr.Blocks(css=css) as demo:
372
- gr.Markdown("# LTX Video 0.9.7 Distilled")
373
- gr.Markdown("Fast high quality video generation. [Model](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-13b-0.9.7-distilled.safetensors) [GitHub](https://github.com/Lightricks/LTX-Video) [Diffusers](#)")
374
-
375
- with gr.Row():
376
- with gr.Column():
377
- with gr.Tab("image-to-video") as image_tab:
378
- video_i_hidden = gr.Textbox(label="video_i", visible=False, value=None)
379
- image_i2v = gr.Image(label="Input Image", type="filepath", sources=["upload", "webcam", "clipboard"])
380
- i2v_prompt = gr.Textbox(label="Prompt", value="The creature from the image starts to move", lines=3)
381
- i2v_button = gr.Button("Generate Image-to-Video", variant="primary")
382
- with gr.Tab("text-to-video") as text_tab:
383
- image_n_hidden = gr.Textbox(label="image_n", visible=False, value=None)
384
- video_n_hidden = gr.Textbox(label="video_n", visible=False, value=None)
385
- t2v_prompt = gr.Textbox(label="Prompt", value="A majestic dragon flying over a medieval castle", lines=3)
386
- t2v_button = gr.Button("Generate Text-to-Video", variant="primary")
387
- with gr.Tab("video-to-video", visible=False) as video_tab:
388
- image_v_hidden = gr.Textbox(label="image_v", visible=False, value=None)
389
- video_v2v = gr.Video(label="Input Video", sources=["upload", "webcam"]) # type defaults to filepath
390
- frames_to_use = gr.Slider(label="Frames to use from input video", minimum=9, maximum=MAX_NUM_FRAMES, value=9, step=8, info="Number of initial frames to use for conditioning/transformation. Must be N*8+1.")
391
- v2v_prompt = gr.Textbox(label="Prompt", value="Change the style to cinematic anime", lines=3)
392
- v2v_button = gr.Button("Generate Video-to-Video", variant="primary")
393
-
394
- duration_input = gr.Slider(
395
- label="Video Duration (seconds)",
396
- minimum=0.3,
397
- maximum=8.5,
398
- value=2,
399
- step=0.1,
400
- info=f"Target video duration (0.3s to 8.5s)"
401
- )
402
- improve_texture = gr.Checkbox(label="Improve Texture (multi-scale)", value=True, info="Uses a two-pass generation for better quality, but is slower. Recommended for final output.")
403
-
404
- with gr.Column():
405
- output_video = gr.Video(label="Generated Video", interactive=False)
406
- # gr.DeepLinkButton()
407
-
408
- with gr.Accordion("Advanced settings", open=False):
409
- mode = gr.Dropdown(["text-to-video", "image-to-video", "video-to-video"], label="task", value="image-to-video", visible=False)
410
- negative_prompt_input = gr.Textbox(label="Negative Prompt", value="worst quality, inconsistent motion, blurry, jittery, distorted", lines=2)
411
- with gr.Row():
412
- seed_input = gr.Number(label="Seed", value=42, precision=0, minimum=0, maximum=2**32-1)
413
- randomize_seed_input = gr.Checkbox(label="Randomize Seed", value=True)
414
- with gr.Row():
415
- guidance_scale_input = gr.Slider(label="Guidance Scale (CFG)", minimum=1.0, maximum=10.0, value=PIPELINE_CONFIG_YAML.get("first_pass", {}).get("guidance_scale", 1.0), step=0.1, info="Controls how much the prompt influences the output. Higher values = stronger influence.")
416
- with gr.Row():
417
- height_input = gr.Slider(label="Height", value=512, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
418
- width_input = gr.Slider(label="Width", value=704, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
419
-
420
- with gr.Accordion("Debug", open=False):
421
- image_i2v_debug = gr.Image(label="Input Image Debug", type="filepath", sources=["upload", "webcam", "clipboard"])
422
- i2v_prompt_debug = gr.Textbox(label="Prompt Debug", value="", lines=3)
423
- height_input_debug = gr.Slider(label="Height Debug", value=512, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
424
- width_input_debug = gr.Slider(label="Width Debug", value=704, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
425
- duration_input_debug = gr.Slider(
426
- label="Video Duration Debug (seconds)",
427
- minimum=0.3,
428
- maximum=8.5,
429
- value=6,
430
- step=0.1,
431
- info=f"Target video duration (0.3s to 8.5s)"
432
- )
433
-
434
- with gr.Row(visible=False):
435
- gr.Examples(
436
- examples = [
437
- [
438
- "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
439
- "",
440
- "./Example_LTX/Example1.png",
441
- None,
442
- 512,
443
- 800,
444
- "image-to-video",
445
- 6,
446
- 9,
447
- 42,
448
- True,
449
- 1,
450
- True
451
- ],
452
- ],
453
- run_on_click = True,
454
- fn = generate,
455
- inputs = [i2v_prompt, negative_prompt_input, image_i2v, video_i_hidden,
456
- height_input, width_input, mode,
457
- duration_input, frames_to_use,
458
- seed_input, randomize_seed_input, guidance_scale_input, improve_texture],
459
- outputs = [output_video, seed_input],
460
- cache_examples = True,
461
- )
462
-
463
- def height_input_debug_change(value):
464
- global height_input_debug_value
465
- height_input_debug_value = value
466
- return []
467
-
468
- def width_input_debug_change(value):
469
- global width_input_debug_value
470
- width_input_debug_value = value
471
- return []
472
-
473
- def duration_input_debug_change(value):
474
- global duration_input_debug_value
475
- duration_input_debug_value = value
476
- return []
477
-
478
- def i2v_prompt_debug_change(prompt):
479
- global i2v_prompt_debug_value
480
- i2v_prompt_debug_value = prompt
481
- return []
482
-
483
- # --- Event handlers for updating dimensions on upload ---
484
- def handle_image_upload_for_dims(image_filepath, current_h, current_w):
485
- if not image_filepath: # Image cleared or no image initially
486
- # Keep current slider values if image is cleared or no input
487
- return gr.update(value=current_h), gr.update(value=current_w)
488
- try:
489
- img = Image.open(image_filepath)
490
- orig_w, orig_h = img.size
491
- new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
492
- return gr.update(value=new_h), gr.update(value=new_w)
493
- except Exception as e:
494
- print(f"Error processing image for dimension update: {e}")
495
- # Keep current slider values on error
496
- return gr.update(value=current_h), gr.update(value=current_w)
497
-
498
- def handle_image_debug_upload_for_dims(image_filepath, current_h, current_w):
499
- global image_i2v_debug_value
500
- image_i2v_debug_value = image_filepath
501
- if not image_filepath: # Image cleared or no image initially
502
- # Keep current slider values if image is cleared or no input
503
- return gr.update(value=current_h), gr.update(value=current_w)
504
- try:
505
- img = Image.open(image_filepath)
506
- orig_w, orig_h = img.size
507
- new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
508
- global height_input_debug_value
509
- height_input_debug_value = new_h
510
- global width_input_debug_value
511
- width_input_debug_value = new_w
512
- return gr.update(value=new_h), gr.update(value=new_w)
513
- except Exception as e:
514
- # Keep current slider values on error
515
- return gr.update(value=current_h), gr.update(value=current_w)
516
-
517
- def handle_video_upload_for_dims(video_filepath, current_h, current_w):
518
- if not video_filepath: # Video cleared or no video initially
519
- return gr.update(value=current_h), gr.update(value=current_w)
520
- try:
521
- # Ensure video_filepath is a string for os.path.exists and imageio
522
- video_filepath_str = str(video_filepath)
523
- if not os.path.exists(video_filepath_str):
524
- print(f"Video file path does not exist for dimension update: {video_filepath_str}")
525
- return gr.update(value=current_h), gr.update(value=current_w)
526
-
527
- orig_w, orig_h = -1, -1
528
- with imageio.get_reader(video_filepath_str) as reader:
529
- meta = reader.get_meta_data()
530
- if 'size' in meta:
531
- orig_w, orig_h = meta['size']
532
- else:
533
- # Fallback: read first frame if 'size' not in metadata
534
- try:
535
- first_frame = reader.get_data(0)
536
- # Shape is (h, w, c) for frames
537
- orig_h, orig_w = first_frame.shape[0], first_frame.shape[1]
538
- except Exception as e_frame:
539
- print(f"Could not get video size from metadata or first frame: {e_frame}")
540
- return gr.update(value=current_h), gr.update(value=current_w)
541
-
542
- if orig_w == -1 or orig_h == -1: # If dimensions couldn't be determined
543
- print(f"Could not determine dimensions for video: {video_filepath_str}")
544
- return gr.update(value=current_h), gr.update(value=current_w)
545
-
546
- new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
547
- return gr.update(value=new_h), gr.update(value=new_w)
548
- except Exception as e:
549
- # Log type of video_filepath for debugging if it's not a path-like string
550
- print(f"Error processing video for dimension update: {e} (Path: {video_filepath}, Type: {type(video_filepath)})")
551
- return gr.update(value=current_h), gr.update(value=current_w)
552
-
553
-
554
- image_i2v_debug.upload(
555
- fn=handle_image_debug_upload_for_dims,
556
- inputs=[image_i2v_debug, height_input_debug, width_input_debug],
557
- outputs=[height_input_debug, width_input_debug]
558
- )
559
-
560
- image_i2v.upload(
561
- fn=handle_image_upload_for_dims,
562
- inputs=[image_i2v, height_input, width_input],
563
- outputs=[height_input, width_input]
564
- )
565
- video_v2v.upload(
566
- fn=handle_video_upload_for_dims,
567
- inputs=[video_v2v, height_input, width_input],
568
- outputs=[height_input, width_input]
569
- )
570
- i2v_prompt_debug.change(
571
- fn=i2v_prompt_debug_change,
572
- inputs=[image_i2v_debug],
573
- outputs=[]
574
- )
575
- height_input_debug.change(
576
- fn=height_input_debug_change,
577
- inputs=[height_input_debug],
578
- outputs=[]
579
- )
580
- width_input_debug.change(
581
- fn=width_input_debug_change,
582
- inputs=[width_input_debug],
583
- outputs=[]
584
- )
585
- duration_input_debug.change(
586
- fn=duration_input_debug_change,
587
- inputs=[duration_input_debug],
588
- outputs=[]
589
- )
590
-
591
- image_tab.select(
592
- fn=update_task_image,
593
- outputs=[mode]
594
- )
595
- text_tab.select(
596
- fn=update_task_text,
597
- outputs=[mode]
598
- )
599
-
600
- t2v_inputs = [t2v_prompt, negative_prompt_input, image_n_hidden, video_n_hidden,
601
- height_input, width_input, mode,
602
- duration_input, frames_to_use,
603
- seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
604
-
605
- i2v_inputs = [i2v_prompt, negative_prompt_input, image_i2v, video_i_hidden,
606
- height_input, width_input, mode,
607
- duration_input, frames_to_use,
608
- seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
609
-
610
- v2v_inputs = [v2v_prompt, negative_prompt_input, image_v_hidden, video_v2v,
611
- height_input, width_input, mode,
612
- duration_input, frames_to_use,
613
- seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
614
-
615
- t2v_button.click(fn=generate, inputs=t2v_inputs, outputs=[output_video, seed_input], api_name="text_to_video")
616
- i2v_button.click(fn=generate, inputs=i2v_inputs, outputs=[output_video, seed_input], api_name="image_to_video")
617
- v2v_button.click(fn=generate, inputs=v2v_inputs, outputs=[output_video, seed_input], api_name="video_to_video")
618
-
619
- if __name__ == "__main__":
620
- if os.path.exists(models_dir) and os.path.isdir(models_dir):
621
- print(f"Model directory: {Path(models_dir).resolve()}")
622
-
623
  demo.queue().launch(debug=True, share=False, mcp_server=True)
 
1
+ import gradio as gr
2
+ import torch
3
+ import spaces
4
+ import numpy as np
5
+ import random
6
+ import os
7
+ import yaml
8
+ from pathlib import Path
9
+ import imageio
10
+ import tempfile
11
+ from PIL import Image
12
+ from huggingface_hub import hf_hub_download
13
+ import shutil
14
+
15
+ from inference import (
16
+ create_ltx_video_pipeline,
17
+ create_latent_upsampler,
18
+ load_image_to_tensor_with_resize_and_crop,
19
+ seed_everething,
20
+ get_device,
21
+ calculate_padding,
22
+ load_media_file
23
+ )
24
+ from ltx_video.pipelines.pipeline_ltx_video import ConditioningItem, LTXMultiScalePipeline, LTXVideoPipeline
25
+ from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
26
+
27
+ image_i2v_debug_value = None
28
+ i2v_prompt_debug_value = None
29
+ height_input_debug_value = None
30
+ width_input_debug_value = None
31
+ duration_input_debug_value = None
32
+ config_file_path = "configs/ltxv-13b-0.9.7-distilled.yaml"
33
+ with open(config_file_path, "r") as file:
34
+ PIPELINE_CONFIG_YAML = yaml.safe_load(file)
35
+
36
+ LTX_REPO = "Lightricks/LTX-Video"
37
+ MAX_IMAGE_SIZE = PIPELINE_CONFIG_YAML.get("max_resolution", 1280)
38
+ MAX_NUM_FRAMES = 257
39
+
40
+ FPS = 30.0
41
+
42
+ # --- Global variables for loaded models ---
43
+ pipeline_instance = None
44
+ latent_upsampler_instance = None
45
+ models_dir = "downloaded_models_gradio_cpu_init"
46
+ Path(models_dir).mkdir(parents=True, exist_ok=True)
47
+
48
+ print("Downloading models (if not present)...")
49
+ distilled_model_actual_path = hf_hub_download(
50
+ repo_id=LTX_REPO,
51
+ filename=PIPELINE_CONFIG_YAML["checkpoint_path"],
52
+ local_dir=models_dir,
53
+ local_dir_use_symlinks=False
54
+ )
55
+ PIPELINE_CONFIG_YAML["checkpoint_path"] = distilled_model_actual_path
56
+ print(f"Distilled model path: {distilled_model_actual_path}")
57
+
58
+ SPATIAL_UPSCALER_FILENAME = PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"]
59
+ spatial_upscaler_actual_path = hf_hub_download(
60
+ repo_id=LTX_REPO,
61
+ filename=SPATIAL_UPSCALER_FILENAME,
62
+ local_dir=models_dir,
63
+ local_dir_use_symlinks=False
64
+ )
65
+ PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"] = spatial_upscaler_actual_path
66
+ print(f"Spatial upscaler model path: {spatial_upscaler_actual_path}")
67
+
68
+ print("Creating LTX Video pipeline on CPU...")
69
+ pipeline_instance = create_ltx_video_pipeline(
70
+ ckpt_path=PIPELINE_CONFIG_YAML["checkpoint_path"],
71
+ precision=PIPELINE_CONFIG_YAML["precision"],
72
+ text_encoder_model_name_or_path=PIPELINE_CONFIG_YAML["text_encoder_model_name_or_path"],
73
+ sampler=PIPELINE_CONFIG_YAML["sampler"],
74
+ device="cpu",
75
+ enhance_prompt=False,
76
+ prompt_enhancer_image_caption_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_image_caption_model_name_or_path"],
77
+ prompt_enhancer_llm_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_llm_model_name_or_path"],
78
+ )
79
+ print("LTX Video pipeline created on CPU.")
80
+
81
+ if PIPELINE_CONFIG_YAML.get("spatial_upscaler_model_path"):
82
+ print("Creating latent upsampler on CPU...")
83
+ latent_upsampler_instance = create_latent_upsampler(
84
+ PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"],
85
+ device="cpu"
86
+ )
87
+ print("Latent upsampler created on CPU.")
88
+
89
+ target_inference_device = "cuda"
90
+ print(f"Target inference device: {target_inference_device}")
91
+ pipeline_instance.to(target_inference_device)
92
+ if latent_upsampler_instance:
93
+ latent_upsampler_instance.to(target_inference_device)
94
+
95
+
96
+ # --- Helper function for dimension calculation ---
97
+ MIN_DIM_SLIDER = 256 # As defined in the sliders minimum attribute
98
+ TARGET_FIXED_SIDE = 768 # Desired fixed side length as per requirement
99
+
100
+ def calculate_new_dimensions(orig_w, orig_h):
101
+ """
102
+ Calculates new dimensions for height and width sliders based on original media dimensions.
103
+ Ensures one side is TARGET_FIXED_SIDE, the other is scaled proportionally,
104
+ both are multiples of 32, and within [MIN_DIM_SLIDER, MAX_IMAGE_SIZE].
105
+ """
106
+ if orig_w == 0 or orig_h == 0:
107
+ # Default to TARGET_FIXED_SIDE square if original dimensions are invalid
108
+ return int(TARGET_FIXED_SIDE), int(TARGET_FIXED_SIDE)
109
+
110
+ if orig_w >= orig_h: # Landscape or square
111
+ new_h = TARGET_FIXED_SIDE
112
+ aspect_ratio = orig_w / orig_h
113
+ new_w_ideal = new_h * aspect_ratio
114
+
115
+ # Round to nearest multiple of 32
116
+ new_w = round(new_w_ideal / 32) * 32
117
+
118
+ # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
119
+ new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
120
+ # Ensure new_h is also clamped (TARGET_FIXED_SIDE should be within these bounds if configured correctly)
121
+ new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE))
122
+ else: # Portrait
123
+ new_w = TARGET_FIXED_SIDE
124
+ aspect_ratio = orig_h / orig_w # Use H/W ratio for portrait scaling
125
+ new_h_ideal = new_w * aspect_ratio
126
+
127
+ # Round to nearest multiple of 32
128
+ new_h = round(new_h_ideal / 32) * 32
129
+
130
+ # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
131
+ new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE))
132
+ # Ensure new_w is also clamped
133
+ new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
134
+
135
+ return int(new_h), int(new_w)
136
+
137
+ def get_duration(prompt, negative_prompt, input_image_filepath, input_video_filepath,
138
+ height_ui, width_ui, mode,
139
+ duration_ui, # Removed ui_steps
140
+ ui_frames_to_use,
141
+ seed_ui, randomize_seed, ui_guidance_scale, improve_texture_flag,
142
+ progress):
143
+ if duration_ui > 7:
144
+ return 75
145
+ else:
146
+ return 60
147
+
148
+ @spaces.GPU(duration=get_duration)
149
+ def generate(prompt, negative_prompt, input_image_filepath, input_video_filepath,
150
+ height_ui, width_ui, mode,
151
+ duration_ui,
152
+ ui_frames_to_use,
153
+ seed_ui, randomize_seed, ui_guidance_scale, improve_texture_flag,
154
+ progress=gr.Progress(track_tqdm=True)):
155
+ global i2v_prompt_debug_value
156
+ global image_i2v_debug_value
157
+ global height_input_debug_value
158
+ global width_input_debug_value
159
+
160
+ if i2v_prompt_debug_value is not None:
161
+ prompt = i2v_prompt_debug_value
162
+ i2v_prompt_debug_value = None
163
+
164
+ if image_i2v_debug_value is not None:
165
+ input_image_filepath = image_i2v_debug_value
166
+ image_i2v_debug_value = None
167
+
168
+ if height_input_debug_value is not None:
169
+ height_ui = height_input_debug_value
170
+ height_input_debug_value = None
171
+
172
+ if width_input_debug_value is not None:
173
+ width_ui = width_input_debug_value
174
+ width_input_debug_value = None
175
+
176
+ if duration_input_debug_value is not None:
177
+ duration_ui = duration_input_debug_value
178
+ global duration_input_debug_value
179
+ duration_input_debug_value = None
180
+
181
+ if randomize_seed:
182
+ seed_ui = random.randint(0, 2**32 - 1)
183
+ seed_everething(int(seed_ui))
184
+
185
+ target_frames_ideal = duration_ui * FPS
186
+ target_frames_rounded = round(target_frames_ideal)
187
+ if target_frames_rounded < 1:
188
+ target_frames_rounded = 1
189
+
190
+ n_val = round((float(target_frames_rounded) - 1.0) / 8.0)
191
+ actual_num_frames = int(n_val * 8 + 1)
192
+
193
+ actual_num_frames = max(9, actual_num_frames)
194
+ actual_num_frames = min(MAX_NUM_FRAMES, actual_num_frames)
195
+
196
+ actual_height = int(height_ui)
197
+ actual_width = int(width_ui)
198
+
199
+ height_padded = ((actual_height - 1) // 32 + 1) * 32
200
+ width_padded = ((actual_width - 1) // 32 + 1) * 32
201
+ num_frames_padded = ((actual_num_frames - 2) // 8 + 1) * 8 + 1
202
+ if num_frames_padded != actual_num_frames:
203
+ print(f"Warning: actual_num_frames ({actual_num_frames}) and num_frames_padded ({num_frames_padded}) differ. Using num_frames_padded for pipeline.")
204
+
205
+ padding_values = calculate_padding(actual_height, actual_width, height_padded, width_padded)
206
+
207
+ call_kwargs = {
208
+ "prompt": prompt,
209
+ "negative_prompt": negative_prompt,
210
+ "height": height_padded,
211
+ "width": width_padded,
212
+ "num_frames": num_frames_padded,
213
+ "frame_rate": int(FPS),
214
+ "generator": torch.Generator(device=target_inference_device).manual_seed(int(seed_ui)),
215
+ "output_type": "pt",
216
+ "conditioning_items": None,
217
+ "media_items": None,
218
+ "decode_timestep": PIPELINE_CONFIG_YAML["decode_timestep"],
219
+ "decode_noise_scale": PIPELINE_CONFIG_YAML["decode_noise_scale"],
220
+ "stochastic_sampling": PIPELINE_CONFIG_YAML["stochastic_sampling"],
221
+ "image_cond_noise_scale": 0.15,
222
+ "is_video": True,
223
+ "vae_per_channel_normalize": True,
224
+ "mixed_precision": (PIPELINE_CONFIG_YAML["precision"] == "mixed_precision"),
225
+ "offload_to_cpu": False,
226
+ "enhance_prompt": False,
227
+ }
228
+
229
+ stg_mode_str = PIPELINE_CONFIG_YAML.get("stg_mode", "attention_values")
230
+ if stg_mode_str.lower() in ["stg_av", "attention_values"]:
231
+ call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionValues
232
+ elif stg_mode_str.lower() in ["stg_as", "attention_skip"]:
233
+ call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionSkip
234
+ elif stg_mode_str.lower() in ["stg_r", "residual"]:
235
+ call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.Residual
236
+ elif stg_mode_str.lower() in ["stg_t", "transformer_block"]:
237
+ call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.TransformerBlock
238
+ else:
239
+ raise ValueError(f"Invalid stg_mode: {stg_mode_str}")
240
+
241
+ if mode == "image-to-video" and input_image_filepath:
242
+ try:
243
+ media_tensor = load_image_to_tensor_with_resize_and_crop(
244
+ input_image_filepath, actual_height, actual_width
245
+ )
246
+ media_tensor = torch.nn.functional.pad(media_tensor, padding_values)
247
+ call_kwargs["conditioning_items"] = [ConditioningItem(media_tensor.to(target_inference_device), 0, 1.0)]
248
+ except Exception as e:
249
+ print(f"Error loading image {input_image_filepath}: {e}")
250
+ raise gr.Error(f"Could not load image: {e}")
251
+ elif mode == "video-to-video" and input_video_filepath:
252
+ try:
253
+ call_kwargs["media_items"] = load_media_file(
254
+ media_path=input_video_filepath,
255
+ height=actual_height,
256
+ width=actual_width,
257
+ max_frames=int(ui_frames_to_use),
258
+ padding=padding_values
259
+ ).to(target_inference_device)
260
+ except Exception as e:
261
+ print(f"Error loading video {input_video_filepath}: {e}")
262
+ raise gr.Error(f"Could not load video: {e}")
263
+
264
+ print(f"Moving models to {target_inference_device} for inference (if not already there)...")
265
+
266
+ active_latent_upsampler = None
267
+ if improve_texture_flag and latent_upsampler_instance:
268
+ active_latent_upsampler = latent_upsampler_instance
269
+
270
+ result_images_tensor = None
271
+ if improve_texture_flag:
272
+ if not active_latent_upsampler:
273
+ raise gr.Error("Spatial upscaler model not loaded or improve_texture not selected, cannot use multi-scale.")
274
+
275
+ multi_scale_pipeline_obj = LTXMultiScalePipeline(pipeline_instance, active_latent_upsampler)
276
+
277
+ first_pass_args = PIPELINE_CONFIG_YAML.get("first_pass", {}).copy()
278
+ first_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
279
+ # num_inference_steps will be derived from len(timesteps) in the pipeline
280
+ first_pass_args.pop("num_inference_steps", None)
281
+
282
+
283
+ second_pass_args = PIPELINE_CONFIG_YAML.get("second_pass", {}).copy()
284
+ second_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
285
+ # num_inference_steps will be derived from len(timesteps) in the pipeline
286
+ second_pass_args.pop("num_inference_steps", None)
287
+
288
+ multi_scale_call_kwargs = call_kwargs.copy()
289
+ multi_scale_call_kwargs.update({
290
+ "downscale_factor": PIPELINE_CONFIG_YAML["downscale_factor"],
291
+ "first_pass": first_pass_args,
292
+ "second_pass": second_pass_args,
293
+ })
294
+
295
+ print(f"Calling multi-scale pipeline (eff. HxW: {actual_height}x{actual_width}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
296
+ result_images_tensor = multi_scale_pipeline_obj(**multi_scale_call_kwargs).images
297
+ else:
298
+ single_pass_call_kwargs = call_kwargs.copy()
299
+ first_pass_config_from_yaml = PIPELINE_CONFIG_YAML.get("first_pass", {})
300
+
301
+ single_pass_call_kwargs["timesteps"] = first_pass_config_from_yaml.get("timesteps")
302
+ single_pass_call_kwargs["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
303
+ single_pass_call_kwargs["stg_scale"] = first_pass_config_from_yaml.get("stg_scale")
304
+ single_pass_call_kwargs["rescaling_scale"] = first_pass_config_from_yaml.get("rescaling_scale")
305
+ single_pass_call_kwargs["skip_block_list"] = first_pass_config_from_yaml.get("skip_block_list")
306
+
307
+ # Remove keys that might conflict or are not used in single pass / handled by above
308
+ single_pass_call_kwargs.pop("num_inference_steps", None)
309
+ single_pass_call_kwargs.pop("first_pass", None)
310
+ single_pass_call_kwargs.pop("second_pass", None)
311
+ single_pass_call_kwargs.pop("downscale_factor", None)
312
+
313
+ print(f"Calling base pipeline (padded HxW: {height_padded}x{width_padded}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
314
+ result_images_tensor = pipeline_instance(**single_pass_call_kwargs).images
315
+
316
+ if result_images_tensor is None:
317
+ raise gr.Error("Generation failed.")
318
+
319
+ pad_left, pad_right, pad_top, pad_bottom = padding_values
320
+ slice_h_end = -pad_bottom if pad_bottom > 0 else None
321
+ slice_w_end = -pad_right if pad_right > 0 else None
322
+
323
+ result_images_tensor = result_images_tensor[
324
+ :, :, :actual_num_frames, pad_top:slice_h_end, pad_left:slice_w_end
325
+ ]
326
+
327
+ video_np = result_images_tensor[0].permute(1, 2, 3, 0).cpu().float().numpy()
328
+
329
+ video_np = np.clip(video_np, 0, 1)
330
+ video_np = (video_np * 255).astype(np.uint8)
331
+
332
+ temp_dir = tempfile.mkdtemp()
333
+ timestamp = random.randint(10000,99999)
334
+ output_video_path = os.path.join(temp_dir, f"output_{timestamp}.mp4")
335
+
336
+ try:
337
+ with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], macro_block_size=1) as video_writer:
338
+ for frame_idx in range(video_np.shape[0]):
339
+ progress(frame_idx / video_np.shape[0], desc="Saving video")
340
+ video_writer.append_data(video_np[frame_idx])
341
+ except Exception as e:
342
+ print(f"Error saving video with macro_block_size=1: {e}")
343
+ try:
344
+ with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], format='FFMPEG', codec='libx264', quality=8) as video_writer:
345
+ for frame_idx in range(video_np.shape[0]):
346
+ progress(frame_idx / video_np.shape[0], desc="Saving video (fallback ffmpeg)")
347
+ video_writer.append_data(video_np[frame_idx])
348
+ except Exception as e2:
349
+ print(f"Fallback video saving error: {e2}")
350
+ raise gr.Error(f"Failed to save video: {e2}")
351
+
352
+ return output_video_path, seed_ui
353
+
354
+ def update_task_image():
355
+ return "image-to-video"
356
+
357
+ def update_task_text():
358
+ return "text-to-video"
359
+
360
+ def update_task_video():
361
+ return "video-to-video"
362
+
363
+ # --- Gradio UI Definition ---
364
+ css="""
365
+ #col-container {
366
+ margin: 0 auto;
367
+ max-width: 900px;
368
+ }
369
+ """
370
+
371
+ with gr.Blocks(css=css) as demo:
372
+ gr.Markdown("# LTX Video 0.9.7 Distilled")
373
+ gr.Markdown("Fast high quality video generation. [Model](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-13b-0.9.7-distilled.safetensors) [GitHub](https://github.com/Lightricks/LTX-Video) [Diffusers](#)")
374
+
375
+ with gr.Row():
376
+ with gr.Column():
377
+ with gr.Tab("image-to-video") as image_tab:
378
+ video_i_hidden = gr.Textbox(label="video_i", visible=False, value=None)
379
+ image_i2v = gr.Image(label="Input Image", type="filepath", sources=["upload", "webcam", "clipboard"])
380
+ i2v_prompt = gr.Textbox(label="Prompt", value="The creature from the image starts to move", lines=3)
381
+ i2v_button = gr.Button("Generate Image-to-Video", variant="primary")
382
+ with gr.Tab("text-to-video") as text_tab:
383
+ image_n_hidden = gr.Textbox(label="image_n", visible=False, value=None)
384
+ video_n_hidden = gr.Textbox(label="video_n", visible=False, value=None)
385
+ t2v_prompt = gr.Textbox(label="Prompt", value="A majestic dragon flying over a medieval castle", lines=3)
386
+ t2v_button = gr.Button("Generate Text-to-Video", variant="primary")
387
+ with gr.Tab("video-to-video", visible=False) as video_tab:
388
+ image_v_hidden = gr.Textbox(label="image_v", visible=False, value=None)
389
+ video_v2v = gr.Video(label="Input Video", sources=["upload", "webcam"]) # type defaults to filepath
390
+ frames_to_use = gr.Slider(label="Frames to use from input video", minimum=9, maximum=MAX_NUM_FRAMES, value=9, step=8, info="Number of initial frames to use for conditioning/transformation. Must be N*8+1.")
391
+ v2v_prompt = gr.Textbox(label="Prompt", value="Change the style to cinematic anime", lines=3)
392
+ v2v_button = gr.Button("Generate Video-to-Video", variant="primary")
393
+
394
+ duration_input = gr.Slider(
395
+ label="Video Duration (seconds)",
396
+ minimum=0.3,
397
+ maximum=8.5,
398
+ value=2,
399
+ step=0.1,
400
+ info=f"Target video duration (0.3s to 8.5s)"
401
+ )
402
+ improve_texture = gr.Checkbox(label="Improve Texture (multi-scale)", value=True, info="Uses a two-pass generation for better quality, but is slower. Recommended for final output.")
403
+
404
+ with gr.Column():
405
+ output_video = gr.Video(label="Generated Video", interactive=False)
406
+ # gr.DeepLinkButton()
407
+
408
+ with gr.Accordion("Advanced settings", open=False):
409
+ mode = gr.Dropdown(["text-to-video", "image-to-video", "video-to-video"], label="task", value="image-to-video", visible=False)
410
+ negative_prompt_input = gr.Textbox(label="Negative Prompt", value="worst quality, inconsistent motion, blurry, jittery, distorted", lines=2)
411
+ with gr.Row():
412
+ seed_input = gr.Number(label="Seed", value=42, precision=0, minimum=0, maximum=2**32-1)
413
+ randomize_seed_input = gr.Checkbox(label="Randomize Seed", value=True)
414
+ with gr.Row():
415
+ guidance_scale_input = gr.Slider(label="Guidance Scale (CFG)", minimum=1.0, maximum=10.0, value=PIPELINE_CONFIG_YAML.get("first_pass", {}).get("guidance_scale", 1.0), step=0.1, info="Controls how much the prompt influences the output. Higher values = stronger influence.")
416
+ with gr.Row():
417
+ height_input = gr.Slider(label="Height", value=512, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
418
+ width_input = gr.Slider(label="Width", value=704, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
419
+
420
+ with gr.Accordion("Debug", open=False):
421
+ image_i2v_debug = gr.Image(label="Input Image Debug", type="filepath", sources=["upload", "webcam", "clipboard"])
422
+ i2v_prompt_debug = gr.Textbox(label="Prompt Debug", value="", lines=3)
423
+ height_input_debug = gr.Slider(label="Height Debug", value=512, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
424
+ width_input_debug = gr.Slider(label="Width Debug", value=704, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
425
+ duration_input_debug = gr.Slider(
426
+ label="Video Duration Debug (seconds)",
427
+ minimum=0.3,
428
+ maximum=8.5,
429
+ value=6,
430
+ step=0.1,
431
+ info=f"Target video duration (0.3s to 8.5s)"
432
+ )
433
+
434
+ with gr.Row(visible=False):
435
+ gr.Examples(
436
+ examples = [
437
+ [
438
+ "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
439
+ "",
440
+ "./Example_LTX/Example1.png",
441
+ None,
442
+ 512,
443
+ 800,
444
+ "image-to-video",
445
+ 6,
446
+ 9,
447
+ 42,
448
+ True,
449
+ 1,
450
+ True
451
+ ],
452
+ ],
453
+ run_on_click = True,
454
+ fn = generate,
455
+ inputs = [i2v_prompt, negative_prompt_input, image_i2v, video_i_hidden,
456
+ height_input, width_input, mode,
457
+ duration_input, frames_to_use,
458
+ seed_input, randomize_seed_input, guidance_scale_input, improve_texture],
459
+ outputs = [output_video, seed_input],
460
+ cache_examples = True,
461
+ )
462
+
463
+ def height_input_debug_change(value):
464
+ global height_input_debug_value
465
+ height_input_debug_value = value
466
+ return []
467
+
468
+ def width_input_debug_change(value):
469
+ global width_input_debug_value
470
+ width_input_debug_value = value
471
+ return []
472
+
473
+ def duration_input_debug_change(value):
474
+ global duration_input_debug_value
475
+ duration_input_debug_value = value
476
+ return []
477
+
478
+ def i2v_prompt_debug_change(prompt):
479
+ global i2v_prompt_debug_value
480
+ i2v_prompt_debug_value = prompt
481
+ return []
482
+
483
+ # --- Event handlers for updating dimensions on upload ---
484
+ def handle_image_upload_for_dims(image_filepath, current_h, current_w):
485
+ if not image_filepath: # Image cleared or no image initially
486
+ # Keep current slider values if image is cleared or no input
487
+ return gr.update(value=current_h), gr.update(value=current_w)
488
+ try:
489
+ img = Image.open(image_filepath)
490
+ orig_w, orig_h = img.size
491
+ new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
492
+ return gr.update(value=new_h), gr.update(value=new_w)
493
+ except Exception as e:
494
+ print(f"Error processing image for dimension update: {e}")
495
+ # Keep current slider values on error
496
+ return gr.update(value=current_h), gr.update(value=current_w)
497
+
498
+ def handle_image_debug_upload_for_dims(image_filepath, current_h, current_w):
499
+ global image_i2v_debug_value
500
+ image_i2v_debug_value = image_filepath
501
+ if not image_filepath: # Image cleared or no image initially
502
+ # Keep current slider values if image is cleared or no input
503
+ return gr.update(value=current_h), gr.update(value=current_w)
504
+ try:
505
+ img = Image.open(image_filepath)
506
+ orig_w, orig_h = img.size
507
+ new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
508
+ global height_input_debug_value
509
+ height_input_debug_value = new_h
510
+ global width_input_debug_value
511
+ width_input_debug_value = new_w
512
+ return gr.update(value=new_h), gr.update(value=new_w)
513
+ except Exception as e:
514
+ # Keep current slider values on error
515
+ return gr.update(value=current_h), gr.update(value=current_w)
516
+
517
+ def handle_video_upload_for_dims(video_filepath, current_h, current_w):
518
+ if not video_filepath: # Video cleared or no video initially
519
+ return gr.update(value=current_h), gr.update(value=current_w)
520
+ try:
521
+ # Ensure video_filepath is a string for os.path.exists and imageio
522
+ video_filepath_str = str(video_filepath)
523
+ if not os.path.exists(video_filepath_str):
524
+ print(f"Video file path does not exist for dimension update: {video_filepath_str}")
525
+ return gr.update(value=current_h), gr.update(value=current_w)
526
+
527
+ orig_w, orig_h = -1, -1
528
+ with imageio.get_reader(video_filepath_str) as reader:
529
+ meta = reader.get_meta_data()
530
+ if 'size' in meta:
531
+ orig_w, orig_h = meta['size']
532
+ else:
533
+ # Fallback: read first frame if 'size' not in metadata
534
+ try:
535
+ first_frame = reader.get_data(0)
536
+ # Shape is (h, w, c) for frames
537
+ orig_h, orig_w = first_frame.shape[0], first_frame.shape[1]
538
+ except Exception as e_frame:
539
+ print(f"Could not get video size from metadata or first frame: {e_frame}")
540
+ return gr.update(value=current_h), gr.update(value=current_w)
541
+
542
+ if orig_w == -1 or orig_h == -1: # If dimensions couldn't be determined
543
+ print(f"Could not determine dimensions for video: {video_filepath_str}")
544
+ return gr.update(value=current_h), gr.update(value=current_w)
545
+
546
+ new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
547
+ return gr.update(value=new_h), gr.update(value=new_w)
548
+ except Exception as e:
549
+ # Log type of video_filepath for debugging if it's not a path-like string
550
+ print(f"Error processing video for dimension update: {e} (Path: {video_filepath}, Type: {type(video_filepath)})")
551
+ return gr.update(value=current_h), gr.update(value=current_w)
552
+
553
+
554
+ image_i2v_debug.upload(
555
+ fn=handle_image_debug_upload_for_dims,
556
+ inputs=[image_i2v_debug, height_input_debug, width_input_debug],
557
+ outputs=[height_input_debug, width_input_debug]
558
+ )
559
+
560
+ image_i2v.upload(
561
+ fn=handle_image_upload_for_dims,
562
+ inputs=[image_i2v, height_input, width_input],
563
+ outputs=[height_input, width_input]
564
+ )
565
+ video_v2v.upload(
566
+ fn=handle_video_upload_for_dims,
567
+ inputs=[video_v2v, height_input, width_input],
568
+ outputs=[height_input, width_input]
569
+ )
570
+ i2v_prompt_debug.change(
571
+ fn=i2v_prompt_debug_change,
572
+ inputs=[image_i2v_debug],
573
+ outputs=[]
574
+ )
575
+ height_input_debug.change(
576
+ fn=height_input_debug_change,
577
+ inputs=[height_input_debug],
578
+ outputs=[]
579
+ )
580
+ width_input_debug.change(
581
+ fn=width_input_debug_change,
582
+ inputs=[width_input_debug],
583
+ outputs=[]
584
+ )
585
+ duration_input_debug.change(
586
+ fn=duration_input_debug_change,
587
+ inputs=[duration_input_debug],
588
+ outputs=[]
589
+ )
590
+
591
+ image_tab.select(
592
+ fn=update_task_image,
593
+ outputs=[mode]
594
+ )
595
+ text_tab.select(
596
+ fn=update_task_text,
597
+ outputs=[mode]
598
+ )
599
+
600
+ t2v_inputs = [t2v_prompt, negative_prompt_input, image_n_hidden, video_n_hidden,
601
+ height_input, width_input, mode,
602
+ duration_input, frames_to_use,
603
+ seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
604
+
605
+ i2v_inputs = [i2v_prompt, negative_prompt_input, image_i2v, video_i_hidden,
606
+ height_input, width_input, mode,
607
+ duration_input, frames_to_use,
608
+ seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
609
+
610
+ v2v_inputs = [v2v_prompt, negative_prompt_input, image_v_hidden, video_v2v,
611
+ height_input, width_input, mode,
612
+ duration_input, frames_to_use,
613
+ seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
614
+
615
+ t2v_button.click(fn=generate, inputs=t2v_inputs, outputs=[output_video, seed_input], api_name="text_to_video")
616
+ i2v_button.click(fn=generate, inputs=i2v_inputs, outputs=[output_video, seed_input], api_name="image_to_video")
617
+ v2v_button.click(fn=generate, inputs=v2v_inputs, outputs=[output_video, seed_input], api_name="video_to_video")
618
+
619
+ if __name__ == "__main__":
620
+ if os.path.exists(models_dir) and os.path.isdir(models_dir):
621
+ print(f"Model directory: {Path(models_dir).resolve()}")
622
+
623
  demo.queue().launch(debug=True, share=False, mcp_server=True)