Buckets:
| """MiniMax-H3 sampling pipeline for ai-toolkit training previews. | |
| Covers t2v (t2va) and first-frame i2v (fl2va), always denoising the joint | |
| audio stream alongside the video (the packed sequence contains audio rows by | |
| construction; decoding the audio track is optional). | |
| MiniMax-H3 is guidance-distilled: there is no negative prompt, no CFG and | |
| exactly one transformer forward per step. ``unconditional_embeds`` and | |
| ``guidance_scale`` are accepted for harness compatibility and ignored. | |
| Scheduler (the released math, not diffusers'): | |
| - sigma grid: ``linspace(1, 0, steps + 1)`` through the exponential shift | |
| (video 12, audio 3), consecutive duplicates collapsed; ``steps`` yields | |
| ``steps`` model evaluations (steps = 1 is one full 1 -> 0 step) | |
| - the model consumes ``t = 1 - sigma`` (t = 1 means clean) and predicts the | |
| data-ward velocity ``clean - noise``: ``denoised = x + sigma * v`` | |
| - Euler update ``x_next = r * x + (1 - r) * denoised`` with | |
| ``r = sigma_next / sigma``, evaluated in float32 | |
| """ | |
| from typing import Optional | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from diffusers.utils.torch_utils import randn_tensor | |
| from . import packing | |
| from .text_encoder import trim_caption_tokens | |
| from .packing import ( | |
| AUDIO_CHANNELS, | |
| AUDIO_SIGMA_SHIFT, | |
| FPS, | |
| KEYFRAME_NOISE_AUG_T, | |
| VIDEO_SIGMA_SHIFT, | |
| build_packed_sequence, | |
| build_row_timesteps, | |
| build_sigma_schedule, | |
| pack_audio_latents, | |
| patchify_video_latents, | |
| remap_sigma, | |
| unpack_audio_tokens, | |
| unpatchify_video_tokens, | |
| ) | |
| class MiniMaxH3Pipeline: | |
| """Lightweight sampler; receives the MinimaxH3Model (BaseModel subclass) | |
| and reuses its VAEs / transformer / device bookkeeping.""" | |
| def __init__(self, model): | |
| self.model = model | |
| def device(self): | |
| return self.model.device_torch | |
| def to(self, *args, **kwargs): | |
| return self | |
| def set_progress_bar_config(self, **kwargs): | |
| pass | |
| def __call__( | |
| self, | |
| conditional_embeds, # AdvancedPromptEmbeds: text_embeds [(L, 5120)], text_token_tags [(L,)] | |
| unconditional_embeds=None, # ignored: MiniMax-H3 is guidance-distilled | |
| height: int = 768, | |
| width: int = 768, | |
| num_frames: int = 124, | |
| num_inference_steps: int = 28, | |
| guidance_scale: float = 1.0, # ignored | |
| latents: Optional[torch.Tensor] = None, | |
| generator: Optional[torch.Generator] = None, | |
| ctrl_img: Optional[ | |
| Image.Image | |
| ] = None, # first-frame keyframe, already canvas-sized | |
| ref_images: Optional[ | |
| list | |
| ] = None, # ref2va references, already area-matched (own aspect, /32) | |
| with_audio: bool = True, | |
| **kwargs, | |
| ): | |
| model = self.model | |
| device = model.device_torch | |
| dtype = model.torch_dtype | |
| transformer = model.transformer | |
| is_video = num_frames > 1 | |
| if is_video: | |
| num_frames = packing.align_num_frames_down(num_frames) | |
| t_lat = packing.video_latent_num_frames(num_frames) | |
| else: | |
| # true single-frame generation (image mode, LTX-2.3 style): one | |
| # latent frame with keyframe-row geometry — the same layout image | |
| # datasets train with, so image LoRAs sample in-distribution | |
| num_frames = 1 | |
| t_lat = 1 | |
| h_lat = height // 16 | |
| w_lat = width // 16 | |
| a_lat = packing.audio_latent_num_frames(num_frames) | |
| text_embeds, token_tags = trim_caption_tokens( | |
| conditional_embeds.text_embeds[0], | |
| conditional_embeds.text_token_tags[0], | |
| getattr(model, "max_text_length", None), | |
| ) | |
| text_embeds = text_embeds.to(device, dtype) | |
| token_tags = token_tags.to("cpu", torch.long) | |
| # --- packed layout ------------------------------------------------- | |
| if ctrl_img is not None and ref_images: | |
| raise ValueError("ctrl_img (first frame) and ref_images are exclusive") | |
| anchors = ("first",) if ctrl_img is not None else () | |
| # references keep their own aspect: latent dims come from each image | |
| # (PIL images are single-frame blocks; video refs arrive as latent | |
| # tensors (C, T, h, w) already encoded by the caller) | |
| ref_blocks = [] | |
| for r in ref_images or []: | |
| if isinstance(r, dict): | |
| lat = r["latent"] | |
| a = r.get("audio_rows") | |
| a_lat = int(a.shape[0]) // 2 if a is not None else 0 | |
| ref_blocks.append((lat.shape[1], lat.shape[2], lat.shape[3], a_lat)) | |
| elif isinstance(r, torch.Tensor): | |
| ref_blocks.append((r.shape[1], r.shape[2], r.shape[3])) | |
| else: | |
| ref_blocks.append((1, r.size[1] // 16, r.size[0] // 16)) | |
| ref_blocks = tuple(ref_blocks) | |
| layout = build_packed_sequence( | |
| text_token_tags=token_tags, | |
| num_latent_frames=t_lat, | |
| latent_height=h_lat, | |
| latent_width=w_lat, | |
| num_audio_latents=a_lat, | |
| keyframe_anchors=anchors, | |
| ref_blocks=ref_blocks, | |
| ) | |
| num_cond = layout.num_condition_video_rows | |
| # --- conditioning rows (draw order: condition noise, video, audio) -- | |
| def encode_condition_image(img: Image.Image) -> torch.Tensor: | |
| cond_noise = randn_tensor( | |
| (1, 24, 1, img.size[1] // 16, img.size[0] // 16), | |
| generator=generator, | |
| dtype=torch.float32, | |
| ).to(device) | |
| frame = torch.from_numpy(np.array(img)).float() | |
| frame = (frame / 255.0) * 2.0 - 1.0 # (H, W, 3) -> [-1, 1] | |
| frame = frame.permute(2, 0, 1)[None, :, None] # (1, 3, 1, H, W) | |
| cond_latents = model.encode_keyframe_latents(frame) # (1, 24, 1, h, w) fp32 | |
| # released noise-aug recipe: x = t * clean + (1 - t) * noise at t = 0.999 | |
| cond_latents = ( | |
| KEYFRAME_NOISE_AUG_T * cond_latents.to(device) | |
| + (1.0 - KEYFRAME_NOISE_AUG_T) * cond_noise | |
| ) | |
| return patchify_video_latents(cond_latents) # (1, rows, 96) | |
| def noise_aug_rows(latents: torch.Tensor) -> torch.Tensor: | |
| cond_noise = randn_tensor( | |
| latents.shape, generator=generator, dtype=torch.float32 | |
| ).to(device) | |
| mixed = ( | |
| KEYFRAME_NOISE_AUG_T * latents.to(device, torch.float32) | |
| + (1.0 - KEYFRAME_NOISE_AUG_T) * cond_noise | |
| ) | |
| return patchify_video_latents(mixed) | |
| cond_rows = None | |
| cond_audio_rows = None | |
| if ctrl_img is not None: | |
| cond_rows = encode_condition_image(ctrl_img) | |
| elif ref_images: | |
| parts = [] | |
| audio_parts = [] | |
| for r in ref_images: | |
| if isinstance(r, dict): | |
| # pre-encoded video reference (+ optional clean soundtrack) | |
| parts.append(noise_aug_rows(r["latent"][None])) | |
| if r.get("audio_rows") is not None: | |
| audio_parts.append(r["audio_rows"][None].to(device)) | |
| elif isinstance(r, torch.Tensor): | |
| parts.append(noise_aug_rows(r[None])) | |
| else: | |
| parts.append(encode_condition_image(r)) | |
| cond_rows = torch.cat(parts, dim=1) | |
| if audio_parts: | |
| cond_audio_rows = torch.cat(audio_parts, dim=1).float() | |
| # --- initial noise ------------------------------------------------- | |
| if latents is None: | |
| latents = randn_tensor( | |
| (1, 24, t_lat, h_lat, w_lat), generator=generator, dtype=torch.float32 | |
| ) | |
| video_rows = patchify_video_latents(latents.to(device).float()) # (1, V, 96) | |
| audio_noise = randn_tensor( | |
| (1, AUDIO_CHANNELS, 32, a_lat), generator=generator, dtype=torch.float32 | |
| ).to(device) | |
| audio_rows = pack_audio_latents(audio_noise) # (1, 2*A, 32) | |
| # --- schedules ----------------------------------------------------- | |
| sigmas_v = build_sigma_schedule( | |
| num_inference_steps, | |
| VIDEO_SIGMA_SHIFT, | |
| t1000_ladder=getattr(model, "t1000_sample_ladder", False), | |
| ).to(device) | |
| # the audio schedule follows the video grid through the closed-form | |
| # shift remap so both streams sit at the same underlying position | |
| sigmas_a = remap_sigma(sigmas_v, VIDEO_SIGMA_SHIFT, AUDIO_SIGMA_SHIFT) | |
| position_ids = layout.position_ids[None].to(device) | |
| tags = layout.token_tags[None].to(device) | |
| video_indices = layout.video_indices.to(device) | |
| audio_indices = layout.audio_indices.to(device) | |
| text_indices = layout.text_indices.to(device) | |
| # --- denoise loop -------------------------------------------------- | |
| num_steps = sigmas_v.shape[0] - 1 | |
| for i in range(num_steps): | |
| sv, sv_next = sigmas_v[i], sigmas_v[i + 1] | |
| sa, sa_next = sigmas_a[i], sigmas_a[i + 1] | |
| t_v = 1.0 - float(sv) | |
| t_a = 1.0 - float(sa) | |
| row_t = build_row_timesteps(layout, t_v, t_a)[None].to(device) | |
| video_in = video_rows | |
| if cond_rows is not None: | |
| video_in = torch.cat([cond_rows, video_rows], dim=1) | |
| audio_in = audio_rows | |
| if cond_audio_rows is not None: | |
| audio_in = torch.cat([cond_audio_rows, audio_rows], dim=1) | |
| video_pred, audio_pred = transformer( | |
| hidden_states=video_in.to(dtype), | |
| audio_hidden_states=audio_in.to(dtype), | |
| encoder_hidden_states=text_embeds[None], | |
| row_timesteps=row_t, | |
| token_tags=tags, | |
| position_ids=position_ids, | |
| video_indices=video_indices, | |
| audio_indices=audio_indices, | |
| text_indices=text_indices, | |
| # target-video token grid (patch 1x2x2); consumed only by VSA models | |
| vsa_video_grid=(t_lat, h_lat // 2, w_lat // 2), | |
| ) | |
| v_video = video_pred[:, num_cond:].float() | |
| v_audio = audio_pred[:, layout.num_condition_audio_rows :].float() | |
| denoised_v = video_rows + sv * v_video | |
| # inference engine preview: the video x0 estimate as (1, C, T, H, W); | |
| # the audio stream is not previewed | |
| emit = getattr(model, "_emit_sample_step", None) | |
| if emit is not None and getattr(model, "sample_step_hook", None) is not None: | |
| emit(unpatchify_video_tokens(denoised_v, t_lat, h_lat, w_lat), i, num_steps) | |
| ratio_v = sv_next / sv | |
| video_rows = ratio_v * video_rows + (1.0 - ratio_v) * denoised_v | |
| denoised_a = audio_rows + sa * v_audio | |
| ratio_a = sa_next / sa if float(sa) != 0.0 else 0.0 | |
| audio_rows = ratio_a * audio_rows + (1.0 - ratio_a) * denoised_a | |
| # --- decode -------------------------------------------------------- | |
| video_latents = unpatchify_video_tokens(video_rows, t_lat, h_lat, w_lat) | |
| video = model.decode_latents(video_latents) # (1, 3, T, H, W) in [-1, 1] | |
| video = ((video.float().clamp(-1, 1) + 1.0) * 127.5).round().to(torch.uint8) | |
| video = video[0].permute(1, 2, 3, 0).cpu() # (T, H, W, C) | |
| if not is_video: | |
| return [Image.fromarray(video[0].numpy())] | |
| audio_out = None | |
| if with_audio: | |
| audio_latents = unpack_audio_tokens(audio_rows, a_lat)[0] # (2, 32, A) | |
| waveform = model.decode_audio_latents( | |
| audio_latents.float() | |
| ) # (2, 1, samples) | |
| audio_out = waveform[:, 0].cpu() # (2, samples) stereo | |
| return { | |
| "video": video, | |
| "fps": FPS, | |
| "audio": audio_out, | |
| "audio_sample_rate": packing.AUDIO_SAMPLE_RATE, | |
| "output_path": None, | |
| } | |
Xet Storage Details
- Size:
- 12.1 kB
- Xet hash:
- 56319b5327adbdee82dfb21adcf52f53b38e32ffd2fbbeb9414b871190cd096b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.