# SPDX-License-Identifier: Apache-2.0 """Host pre-processing of RDT2-FM, re-implemented from the pinned sources (no transformers at run time). Official path (thu-ml/RDT2 models/rdt_inferencer.py ``prepreare_data_from_vision_language_model``): images = [obs[cam] for cam in camera_names] # ["left_stereo", "right_stereo"] image = np.concatenate(images, axis=1) # (384, 768, 3) uint8, left | right text = chat_template([{user: [image, instruction]}], add_generation_prompt=False) + "<|im_start|>assistant\n<|quad_start|>" inputs = Qwen2.5-VL processor (use_fast=True, padding_side="left")(text, [image]) Processor (transformers 4.51.3 Qwen2VLImageProcessorFast + Qwen2_5_VLProcessor): smart_resize(384, 768, factor=28) -> (392, 756); torchvision bicubic+antialias resize of the uint8 CHW tensor (== torch.nn.functional.interpolate on uint8, same kernel); x = (x - 255*mean) / (255*std) in fp32; the single frame is repeated to temporal_patch_size=2; patches ordered (t, h/2, w/2, 2, 2) x (C, T, 14, 14) -> (1512, 1176); grid_thw = (1, 28, 54); "<|image_pad|>" expanded to 1512 / 4 = 378 tokens. """ from __future__ import annotations import math import os from functools import lru_cache from typing import Dict, List, Optional, Sequence, Tuple import numpy as np import torch import torch.nn.functional as F from .config import CONFIG, DEFAULT_DIRS, IMAGE_TOKEN_ID, VISION_START_ID V = CONFIG.vision P = CONFIG.proc LM = CONFIG.lm # --------------------------------------------------------------------------------------------------------------- # Image # --------------------------------------------------------------------------------------------------------------- def smart_resize(height: int, width: int, factor: int = 28, min_pixels: int = P.min_pixels, max_pixels: int = P.max_pixels) -> Tuple[int, int]: if height < factor or width < factor: raise ValueError(f"height:{height} and width:{width} must be larger than factor:{factor}") if max(height, width) / min(height, width) > 200: raise ValueError("absolute aspect ratio must be smaller than 200") h_bar = round(height / factor) * factor w_bar = round(width / factor) * factor if h_bar * w_bar > max_pixels: beta = math.sqrt((height * width) / max_pixels) h_bar = math.floor(height / beta / factor) * factor w_bar = math.floor(width / beta / factor) * factor elif h_bar * w_bar < min_pixels: beta = math.sqrt(min_pixels / (height * width)) h_bar = math.ceil(height * beta / factor) * factor w_bar = math.ceil(width * beta / factor) * factor return h_bar, w_bar def concat_cameras(images: Sequence[np.ndarray]) -> np.ndarray: """Cameras in model order (left_stereo, right_stereo), each (H, W, 3) uint8 -> (H, sum W, 3).""" for im in images: if im.dtype != np.uint8 or im.ndim != 3 or im.shape[2] != 3: raise ValueError(f"expected HxWx3 uint8 images, got {im.dtype} {im.shape}") return np.ascontiguousarray(np.concatenate(images, axis=1)) def image_to_patches(image: np.ndarray) -> Tuple[torch.Tensor, torch.Tensor]: """(H, W, 3) uint8 -> pixel_values (N, 1176) fp32, grid_thw (1, 3) int64 (exactly the fast processor's output).""" x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None] # (1, 3, H, W) uint8 h, w = x.shape[-2:] rh, rw = smart_resize(h, w, factor=V.patch * V.merge) if (rh, rw) != (h, w): x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False) inv = 1.0 / P.rescale_factor mean = torch.tensor(P.image_mean) * inv std = torch.tensor(P.image_std) * inv x = (x.to(torch.float32) - mean.view(1, 3, 1, 1)) / std.view(1, 3, 1, 1) x = x.repeat(V.temporal_patch, 1, 1, 1) # (T=2, 3, H, W): single frame repeated gt, gh, gw = 1, rh // V.patch, rw // V.patch x = x.view(gt, V.temporal_patch, 3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch) x = x.permute(0, 3, 6, 4, 7, 2, 1, 5, 8) pv = x.reshape(gt * gh * gw, 3 * V.temporal_patch * V.patch * V.patch) return pv.contiguous(), torch.tensor([[gt, gh, gw]], dtype=torch.int64) # --------------------------------------------------------------------------------------------------------------- # Prompt / tokens # --------------------------------------------------------------------------------------------------------------- def build_prompt(instruction: str, num_image_tokens: int) -> str: """Official text (chat template with one user turn [image, text], no generation prompt, + the guidance suffix), with "<|image_pad|>" already expanded the way Qwen2_5_VLProcessor does.""" return ("<|im_start|>system\n" + P.system_prompt + "<|im_end|>\n<|im_start|>user\n<|vision_start|>" + "<|image_pad|>" * num_image_tokens + "<|vision_end|>" + instruction + "<|im_end|>\n" + P.suffix) @lru_cache(maxsize=2) def _tokenizer(processor_dir: str): from tokenizers import Tokenizer return Tokenizer.from_file(os.path.join(processor_dir, "tokenizer.json")) @lru_cache(maxsize=256) def _token_ids(instruction: str, num_image_tokens: int, processor_dir: str) -> Tuple[int, ...]: tok = _tokenizer(processor_dir) return tuple(tok.encode(build_prompt(instruction, num_image_tokens), add_special_tokens=True).ids) def tokenize(instruction: str, num_image_tokens: int, processor_dir: Optional[str] = None) -> torch.Tensor: """(1, n) int64 prompt ids. Deterministic, so the ids are cached per (instruction, image tokens, processor): a served request tokenizes its instruction twice (bucket choice, then the inputs) and robots repeat instructions.""" ids = _token_ids(str(instruction), int(num_image_tokens), processor_dir or DEFAULT_DIRS["processor"]) return torch.tensor([ids], dtype=torch.int64) # --------------------------------------------------------------------------------------------------------------- # Positions # --------------------------------------------------------------------------------------------------------------- def mrope_position_ids(input_ids: torch.Tensor, image_grid_thw: torch.Tensor) -> torch.Tensor: """3D mRoPE positions (3, 1, L) of Qwen2.5-VL ``get_rope_index`` for batch 1, images only, no padding.""" ids = input_ids[0].tolist() L = len(ids) pos = [] st = 0 for t, h, w in image_grid_thw.tolist(): ed = ids.index(IMAGE_TOKEN_ID, st) st_idx = (pos[-1].max().item() + 1) if pos else 0 text_len = ed - st pos.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) gh, gw = h // V.merge, w // V.merge t_index = torch.arange(t).view(-1, 1).expand(-1, gh * gw).flatten() * 0 # images: second_per_grid_t = 0 h_index = torch.arange(gh).view(1, -1, 1).expand(t, -1, gw).flatten() w_index = torch.arange(gw).view(1, 1, -1).expand(t, gh, -1).flatten() pos.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) st = ed + t * gh * gw if st < L: st_idx = (pos[-1].max().item() + 1) if pos else 0 pos.append(torch.arange(L - st).view(1, -1).expand(3, -1) + st_idx) return torch.cat(pos, dim=1).reshape(3, 1, L).to(torch.int64) def mrope_cos_sin(position_ids: torch.Tensor, theta: float = LM.rope_theta, head_dim: int = LM.head_dim, section: Sequence[int] = LM.mrope_section) -> Tuple[torch.Tensor, torch.Tensor]: """cos/sin (L, head_dim) fp32 with the mRoPE section selection already applied (t | h | w per frequency band).""" inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.int64).float() / head_dim)) # (64,) freqs = position_ids[:, 0, :, None].float() * inv_freq[None, None, :] # (3, L, 64) emb = torch.cat([freqs, freqs], dim=-1) # (3, L, 128) cos, sin = emb.cos(), emb.sin() sec = list(section) * 2 cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(sec, dim=-1))], dim=-1) sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(sec, dim=-1))], dim=-1) return cos, sin # --------------------------------------------------------------------------------------------------------------- # Vision geometry: window order, cu_seqlens and 2D rotary of the ViT # --------------------------------------------------------------------------------------------------------------- def vision_window_index(grid_thw: torch.Tensor) -> Tuple[torch.Tensor, List[int]]: """(window_index over merged tokens, cu_window_seqlens over patches, deduplicated).""" win = V.window // V.merge // V.patch # 4 merged tokens per window side window_index, cu = [], [0] base = 0 for t, h, w in grid_thw.tolist(): gh, gw = h // V.merge, w // V.merge index = torch.arange(t * gh * gw).reshape(t, gh, gw) pad_h = win - gh % win pad_w = win - gw % win nh, nw = (gh + pad_h) // win, (gw + pad_w) // win ip = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100) ip = ip.reshape(t, nh, win, nw, win).permute(0, 1, 3, 2, 4).reshape(t, nh * nw, win, win) seqlens = (ip != -100).sum([2, 3]).reshape(-1) ip = ip.reshape(-1) window_index.append(ip[ip != -100] + base) cu.extend((seqlens.cumsum(0) * V.merge ** 2 + cu[-1]).tolist()) base += t * gh * gw cu_dedup = [cu[0]] for c in cu[1:]: if c != cu_dedup[-1]: cu_dedup.append(c) return torch.cat(window_index), cu_dedup def vision_full_seqlens(grid_thw: torch.Tensor) -> List[int]: cu = [0] for t, h, w in grid_thw.tolist(): for _ in range(t): cu.append(cu[-1] + h * w) return cu def vision_rotary(grid_thw: torch.Tensor) -> torch.Tensor: """(N, 40) rotary angles in raster-merge order (before the window permutation).""" pos_ids = [] for t, h, w in grid_thw.tolist(): hpos = torch.arange(h).unsqueeze(1).expand(-1, w) hpos = hpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten() wpos = torch.arange(w).unsqueeze(0).expand(h, -1) wpos = wpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten() pos_ids.append(torch.stack([hpos, wpos], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) dim = V.head_dim // 2 # 40 inv_freq = 1.0 / (V.rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) # (20,) max_grid = int(grid_thw[:, 1:].max()) full = torch.outer(torch.arange(max_grid, dtype=torch.float), inv_freq) return full[pos_ids].flatten(1) def vision_geometry(grid_thw: torch.Tensor) -> Dict[str, object]: """Everything input-independent the ViT needs for one image geometry (static per preset).""" n = int(grid_thw.prod(-1).sum()) unit = V.merge ** 2 window_index, cu_window = vision_window_index(grid_thw) rot = vision_rotary(grid_thw).reshape(n // unit, unit, -1)[window_index].reshape(n, -1) emb = torch.cat([rot, rot], dim=-1) patch_perm = (window_index[:, None] * unit + torch.arange(unit)[None]).reshape(-1) # windowed <- raster return dict(n=n, window_index=window_index, patch_perm=patch_perm, reverse=torch.argsort(window_index), cu_window=cu_window, cu_full=vision_full_seqlens(grid_thw), cos=emb.cos(), sin=emb.sin()) # --------------------------------------------------------------------------------------------------------------- # Whole request # --------------------------------------------------------------------------------------------------------------- def preprocess(images: Sequence[np.ndarray], instruction: str, processor_dir: Optional[str] = None) -> Dict[str, torch.Tensor]: """images in model order (left_stereo, right_stereo). Returns the VLM inputs of one request (batch 1).""" image = concat_cameras(images) pixel_values, grid_thw = image_to_patches(image) n_img_tok = int(grid_thw.prod()) // (V.merge ** 2) input_ids = tokenize(instruction, n_img_tok, processor_dir) if int((input_ids == IMAGE_TOKEN_ID).sum()) != n_img_tok: raise RuntimeError("image token expansion mismatch") if VISION_START_ID not in input_ids[0].tolist(): raise RuntimeError("missing <|vision_start|>") position_ids = mrope_position_ids(input_ids, grid_thw) return dict(image=torch.from_numpy(image), pixel_values=pixel_values, image_grid_thw=grid_thw, input_ids=input_ids, attention_mask=torch.ones_like(input_ids), position_ids=position_ids) # --------------------------------------------------------------------------------------------------------------- # Static-shape bucketing (TT port): right padding # --------------------------------------------------------------------------------------------------------------- PROMPT_BUCKETS = (416, 448, 512) def choose_bucket(n_tokens: int, buckets: Sequence[int] = PROMPT_BUCKETS) -> int: for b in buckets: if n_tokens <= b: return b raise ValueError(f"prompt of {n_tokens} tokens exceeds the largest bucket {buckets[-1]} (never truncated)") def bucket_inputs(inputs: Dict[str, torch.Tensor], bucket: Optional[int] = None) -> Dict[str, torch.Tensor]: """Right-pad a request to a static prompt bucket. Pad ids = <|endoftext|> (151643), pad positions continue the text positions (max + 1 + j on all 3 axes), key mask False on the pads. Causal attention means the valid rows (and their K/V) are unchanged; the expert masks pad keys. """ from .config import PAD_TOKEN_ID ids, pos = inputs["input_ids"], inputs["position_ids"] L = ids.shape[1] B = bucket or choose_bucket(L) n = B - L out = dict(inputs) out["input_ids"] = torch.cat([ids, torch.full((1, n), PAD_TOKEN_ID, dtype=ids.dtype)], dim=1) start = int(pos.max()) + 1 pad_pos = (torch.arange(n) + start).view(1, 1, n).expand(3, 1, n) out["position_ids"] = torch.cat([pos, pad_pos], dim=2) out["attention_mask"] = torch.cat([inputs["attention_mask"], torch.zeros((1, n), dtype=inputs["attention_mask"].dtype)], 1) out["valid_len"] = torch.tensor(L) return out