Buckets:
| """Qwen3-VL conditioning for MiniMax-H3. | |
| MiniMax-H3 conditions on the **unnormalized** ``hidden_states[50]`` of its | |
| Qwen3-VL-32B conditioner (``hidden_states[0]`` is the embedding output, so | |
| this is the output of decoder layer 49, before the final norm). The LM head | |
| and layers 50..63 are never used, which lets the loader truncate the stack. | |
| The presentation is raw tokens — no chat template, no special tokens: | |
| - t2va: the verbatim prompt. | |
| - fl2va: per keyframe, a ``"<Picture i>: "`` label plus a vision block | |
| (``<|vision_start|>`` + one ``<|image_pad|>`` per merged vision patch + | |
| ``<|vision_end|>``), then the verbatim prompt. Vision-block rows are | |
| tagged as *video* (0) rather than text (1) — the transformer's AdaLN | |
| modality selection keys off these tags. | |
| """ | |
| from typing import List, Optional | |
| import torch | |
| from dataclasses import dataclass, field | |
| from .packing import TEXT_TAG, VIDEO_TAG | |
| TEXT_ENCODER_LAYER = 50 | |
| class VideoRef: | |
| """A reference VIDEO for the Qwen3-VL presentation: frames sampled at | |
| 2 fps with their timestamps (seconds). Presented ComfyUI-style as | |
| ``<Video k>: `` plus one ``<T.T seconds>``-stamped vision block per | |
| merged frame pair.""" | |
| frames: list = field(default_factory=list) # PIL images | |
| timestamps: list = field(default_factory=list) # float seconds, per frame | |
| # a soundtrack that rides as reference audio rows: the presentation gets an | |
| # "<Audio j>: " label emitted BEFORE the "<Video k>: " block (audio itself | |
| # never enters Qwen) | |
| has_audio: bool = False | |
| def video_has_audio(path) -> bool: | |
| try: | |
| import av | |
| with av.open(path) as c: | |
| return len(c.streams.audio) > 0 | |
| except Exception: | |
| return False | |
| def load_video_ref(path, max_frames: int = 0, has_audio=None) -> "VideoRef": | |
| """Sample a video at 2 fps (slot rounding on its native fps) into a | |
| VideoRef with per-frame timestamps in seconds.""" | |
| import cv2 | |
| from PIL import Image as _Image | |
| cap = cv2.VideoCapture(path) | |
| if not cap.isOpened(): | |
| raise ValueError(f"Could not open control video {path}") | |
| fps = cap.get(cv2.CAP_PROP_FPS) or 24.0 | |
| total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) | |
| indices = [] | |
| i = 0 | |
| while True: | |
| idx = round(i * fps / 2.0) | |
| if idx >= total: | |
| break | |
| if not indices or idx != indices[-1]: | |
| indices.append(idx) | |
| i += 1 | |
| if max_frames and len(indices) >= max_frames: | |
| break | |
| frames, times = [], [] | |
| for idx in indices: | |
| cap.set(cv2.CAP_PROP_POS_FRAMES, idx) | |
| ok, frame = cap.read() | |
| if not ok: | |
| break | |
| frames.append(_Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))) | |
| times.append(idx / fps) | |
| cap.release() | |
| if not frames: | |
| raise ValueError(f"No frames decoded from control video {path}") | |
| if has_audio is None: | |
| has_audio = video_has_audio(path) | |
| return VideoRef(frames=frames, timestamps=times, has_audio=has_audio) | |
| def trim_caption_tokens(embeds: torch.Tensor, tags: torch.Tensor, max_length): | |
| """Cap the CAPTION rows of an already-encoded prompt at ``max_length``. | |
| The presentation is ``[labels + vision blocks ...] + caption``: everything | |
| up to and including the last vision row (tag 0) is structural conditioning | |
| and kept intact; only the trailing text-tagged caption tail is truncated. | |
| Lets embeds cached with a longer max_text_length serve a shorter one | |
| without re-encoding. Returns (embeds, tags) — same objects when nothing to | |
| trim. | |
| """ | |
| if max_length is None or max_length <= 0: | |
| return embeds, tags | |
| is_vision = tags.to("cpu") == VIDEO_TAG | |
| if bool(is_vision.any()): | |
| head = int(is_vision.nonzero()[-1].item()) + 1 | |
| else: | |
| head = 0 | |
| caption_len = int(tags.shape[0]) - head | |
| if caption_len <= max_length: | |
| return embeds, tags | |
| keep = head + max_length | |
| return embeds[:keep], tags[:keep] | |
| def encode_minimax_h3_prompt( | |
| text_encoder, # transformers Qwen3VLForConditionalGeneration | |
| tokenizer, # Qwen2TokenizerFast | |
| processor, # Qwen3VLProcessor (needed only when keyframes are present) | |
| prompt: str, | |
| keyframes: Optional[List] = None, # PIL images already on the target canvas | |
| device: Optional[torch.device] = None, | |
| dtype: Optional[torch.dtype] = None, | |
| max_length: Optional[ | |
| int | |
| ] = None, # cap on PROMPT tokens (vision blocks are never cut) | |
| ): | |
| """Encode ONE prompt (with optional keyframes) into MiniMax-H3 conditioning. | |
| Returns (embeds (L, 5120), token_tags (L,) long). The embeds come from | |
| ``hidden_states[50]`` unnormalized. A stack truncated to exactly 50 layers | |
| also works ONLY if the final ``model.norm`` has been replaced with an | |
| Identity (transformers applies the final norm to the last entry of | |
| ``hidden_states``); the loader in minimax_h3.py does exactly that. | |
| """ | |
| num_layers = text_encoder.config.text_config.num_hidden_layers | |
| if num_layers < TEXT_ENCODER_LAYER: | |
| raise ValueError( | |
| f"MiniMax-H3 needs at least {TEXT_ENCODER_LAYER} Qwen3-VL decoder " | |
| f"layers to read hidden_states[{TEXT_ENCODER_LAYER}], got {num_layers}" | |
| ) | |
| if device is None: | |
| device = text_encoder.device | |
| pixel_values, image_grid_thw = None, None | |
| pixel_values_videos, video_grid_thw = None, None | |
| token_ids: List[int] = [] | |
| token_tags: List[int] = [] | |
| if keyframes: | |
| images = [k for k in keyframes if not isinstance(k, VideoRef)] | |
| videos = [k for k in keyframes if isinstance(k, VideoRef)] | |
| merge = processor.image_processor.merge_size**2 | |
| vision_start = tokenizer.convert_tokens_to_ids("<|vision_start|>") | |
| vision_end = tokenizer.convert_tokens_to_ids("<|vision_end|>") | |
| image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>") | |
| video_pad = tokenizer.convert_tokens_to_ids("<|video_pad|>") | |
| if images: | |
| vision = processor.image_processor(images=images, return_tensors="pt") | |
| pixel_values = vision["pixel_values"] | |
| image_grid_thw = vision["image_grid_thw"] | |
| if videos: | |
| import numpy as np | |
| # frames are already sampled at 2 fps: do_sample_frames=False keeps | |
| # the video processor from resampling them (official diffusers | |
| # ref2va encoder passes the same flag) | |
| vids = processor.video_processor( | |
| videos=[np.stack([np.asarray(f.convert("RGB")) for f in v.frames]) for v in videos], | |
| do_sample_frames=False, | |
| return_tensors="pt", | |
| ) | |
| pixel_values_videos = vids["pixel_values_videos"] | |
| video_grid_thw = vids["video_grid_thw"] | |
| pic_idx, vid_idx, aud_idx = 0, 0, 0 | |
| for k in keyframes: | |
| if isinstance(k, VideoRef): | |
| grid = video_grid_thw[vid_idx] | |
| per_pair = int(grid[1] * grid[2]) // merge | |
| label_ids = [] | |
| if k.has_audio: | |
| aud_idx += 1 | |
| label_ids += tokenizer( | |
| f"<Audio {aud_idx}>: ", add_special_tokens=False | |
| )["input_ids"] | |
| label_ids += tokenizer( | |
| f"<Video {vid_idx + 1}>: ", add_special_tokens=False | |
| )["input_ids"] | |
| token_ids += label_ids | |
| token_tags += [TEXT_TAG] * len(label_ids) | |
| # one timestamped vision block per merged frame PAIR: the | |
| # video processor merges temporal_patch_size=2 frames, repeat- | |
| # padding an odd count; the timestamp is the pair's mean time | |
| times = list(k.timestamps) | |
| if len(times) % 2 == 1: | |
| times.append(times[-1]) | |
| for t_pair in range(int(grid[0])): | |
| mean_t = (times[2 * t_pair] + times[2 * t_pair + 1]) / 2.0 | |
| ts_ids = tokenizer( | |
| f"<{round(mean_t, 1):.1f} seconds>", add_special_tokens=False | |
| )["input_ids"] | |
| vision_ids = [vision_start] + [video_pad] * per_pair + [vision_end] | |
| token_ids += ts_ids + vision_ids | |
| token_tags += [TEXT_TAG] * len(ts_ids) + [VIDEO_TAG] * len( | |
| vision_ids | |
| ) | |
| vid_idx += 1 | |
| else: | |
| num_image_tokens = int(image_grid_thw[pic_idx].prod()) // merge | |
| label_ids = tokenizer( | |
| f"<Picture {pic_idx + 1}>: ", add_special_tokens=False | |
| )["input_ids"] | |
| vision_ids = ( | |
| [vision_start] + [image_pad] * num_image_tokens + [vision_end] | |
| ) | |
| token_ids += label_ids + vision_ids | |
| token_tags += [TEXT_TAG] * len(label_ids) + [VIDEO_TAG] * len( | |
| vision_ids | |
| ) | |
| pic_idx += 1 | |
| prompt_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"] | |
| if max_length is not None and max_length > 0: | |
| # the cap applies to the caption only; a keyframe's vision block is | |
| # structural conditioning and cannot be truncated without corrupting it | |
| prompt_ids = prompt_ids[:max_length] | |
| token_ids += prompt_ids | |
| token_tags += [TEXT_TAG] * len(prompt_ids) | |
| if len(token_ids) == 0: | |
| # empty (unconditional) prompt: a single pad token keeps the sequence | |
| # non-degenerate; the model was not trained with CFG so this is only | |
| # ever a fallback | |
| token_ids = [tokenizer.pad_token_id or 0] | |
| token_tags = [TEXT_TAG] | |
| input_ids = torch.tensor([token_ids], dtype=torch.long, device=device) | |
| mm_token_type_ids = torch.tensor( | |
| processor.create_mm_token_type_ids([token_ids]), dtype=torch.long, device=device | |
| ) | |
| # call the inner .model directly: the LM head's vocab projection is dead | |
| # weight here and hidden_states[50] is all that is consumed | |
| outputs = text_encoder.model( | |
| input_ids=input_ids, | |
| attention_mask=torch.ones_like(input_ids), | |
| mm_token_type_ids=mm_token_type_ids, | |
| pixel_values=None | |
| if pixel_values is None | |
| else pixel_values.to(device, text_encoder.dtype), | |
| image_grid_thw=None if image_grid_thw is None else image_grid_thw.to(device), | |
| pixel_values_videos=None | |
| if pixel_values_videos is None | |
| else pixel_values_videos.to(device, text_encoder.dtype), | |
| video_grid_thw=None if video_grid_thw is None else video_grid_thw.to(device), | |
| use_cache=False, | |
| output_hidden_states=True, | |
| ) | |
| layer = min(TEXT_ENCODER_LAYER, len(outputs.hidden_states) - 1) | |
| embeds = outputs.hidden_states[layer][0] | |
| if dtype is not None: | |
| embeds = embeds.to(dtype) | |
| return embeds, torch.tensor(token_tags, dtype=torch.long) | |
Xet Storage Details
- Size:
- 11 kB
- Xet hash:
- 1156f2b28a932f8d1d6ad09e2bde60af287b483db1e27c30abfb52db2c1352f5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.