rdt2-base-p150 / code /tt_rdt2 /reference /preprocess.py
changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
14.2 kB
# SPDX-License-Identifier: Apache-2.0
"""Host pre-processing of RDT2-FM, re-implemented from the pinned sources (no transformers at run time).
Official path (thu-ml/RDT2 models/rdt_inferencer.py ``prepreare_data_from_vision_language_model``):
images = [obs[cam] for cam in camera_names] # ["left_stereo", "right_stereo"]
image = np.concatenate(images, axis=1) # (384, 768, 3) uint8, left | right
text = chat_template([{user: [image, instruction]}], add_generation_prompt=False)
+ "<|im_start|>assistant\n<|quad_start|>"
inputs = Qwen2.5-VL processor (use_fast=True, padding_side="left")(text, [image])
Processor (transformers 4.51.3 Qwen2VLImageProcessorFast + Qwen2_5_VLProcessor):
smart_resize(384, 768, factor=28) -> (392, 756); torchvision bicubic+antialias resize of the uint8 CHW tensor
(== torch.nn.functional.interpolate on uint8, same kernel); x = (x - 255*mean) / (255*std) in fp32; the single frame
is repeated to temporal_patch_size=2; patches ordered (t, h/2, w/2, 2, 2) x (C, T, 14, 14) -> (1512, 1176);
grid_thw = (1, 28, 54); "<|image_pad|>" expanded to 1512 / 4 = 378 tokens.
"""
from __future__ import annotations
import math
import os
from functools import lru_cache
from typing import Dict, List, Optional, Sequence, Tuple
import numpy as np
import torch
import torch.nn.functional as F
from .config import CONFIG, DEFAULT_DIRS, IMAGE_TOKEN_ID, VISION_START_ID
V = CONFIG.vision
P = CONFIG.proc
LM = CONFIG.lm
# ---------------------------------------------------------------------------------------------------------------
# Image
# ---------------------------------------------------------------------------------------------------------------
def smart_resize(height: int, width: int, factor: int = 28, min_pixels: int = P.min_pixels,
max_pixels: int = P.max_pixels) -> Tuple[int, int]:
if height < factor or width < factor:
raise ValueError(f"height:{height} and width:{width} must be larger than factor:{factor}")
if max(height, width) / min(height, width) > 200:
raise ValueError("absolute aspect ratio must be smaller than 200")
h_bar = round(height / factor) * factor
w_bar = round(width / factor) * factor
if h_bar * w_bar > max_pixels:
beta = math.sqrt((height * width) / max_pixels)
h_bar = math.floor(height / beta / factor) * factor
w_bar = math.floor(width / beta / factor) * factor
elif h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (height * width))
h_bar = math.ceil(height * beta / factor) * factor
w_bar = math.ceil(width * beta / factor) * factor
return h_bar, w_bar
def concat_cameras(images: Sequence[np.ndarray]) -> np.ndarray:
"""Cameras in model order (left_stereo, right_stereo), each (H, W, 3) uint8 -> (H, sum W, 3)."""
for im in images:
if im.dtype != np.uint8 or im.ndim != 3 or im.shape[2] != 3:
raise ValueError(f"expected HxWx3 uint8 images, got {im.dtype} {im.shape}")
return np.ascontiguousarray(np.concatenate(images, axis=1))
def image_to_patches(image: np.ndarray) -> Tuple[torch.Tensor, torch.Tensor]:
"""(H, W, 3) uint8 -> pixel_values (N, 1176) fp32, grid_thw (1, 3) int64 (exactly the fast processor's output)."""
x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None] # (1, 3, H, W) uint8
h, w = x.shape[-2:]
rh, rw = smart_resize(h, w, factor=V.patch * V.merge)
if (rh, rw) != (h, w):
x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False)
inv = 1.0 / P.rescale_factor
mean = torch.tensor(P.image_mean) * inv
std = torch.tensor(P.image_std) * inv
x = (x.to(torch.float32) - mean.view(1, 3, 1, 1)) / std.view(1, 3, 1, 1)
x = x.repeat(V.temporal_patch, 1, 1, 1) # (T=2, 3, H, W): single frame repeated
gt, gh, gw = 1, rh // V.patch, rw // V.patch
x = x.view(gt, V.temporal_patch, 3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch)
x = x.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)
pv = x.reshape(gt * gh * gw, 3 * V.temporal_patch * V.patch * V.patch)
return pv.contiguous(), torch.tensor([[gt, gh, gw]], dtype=torch.int64)
# ---------------------------------------------------------------------------------------------------------------
# Prompt / tokens
# ---------------------------------------------------------------------------------------------------------------
def build_prompt(instruction: str, num_image_tokens: int) -> str:
"""Official text (chat template with one user turn [image, text], no generation prompt, + the guidance suffix),
with "<|image_pad|>" already expanded the way Qwen2_5_VLProcessor does."""
return ("<|im_start|>system\n" + P.system_prompt + "<|im_end|>\n<|im_start|>user\n<|vision_start|>"
+ "<|image_pad|>" * num_image_tokens + "<|vision_end|>" + instruction + "<|im_end|>\n" + P.suffix)
@lru_cache(maxsize=2)
def _tokenizer(processor_dir: str):
from tokenizers import Tokenizer
return Tokenizer.from_file(os.path.join(processor_dir, "tokenizer.json"))
@lru_cache(maxsize=256)
def _token_ids(instruction: str, num_image_tokens: int, processor_dir: str) -> Tuple[int, ...]:
tok = _tokenizer(processor_dir)
return tuple(tok.encode(build_prompt(instruction, num_image_tokens), add_special_tokens=True).ids)
def tokenize(instruction: str, num_image_tokens: int, processor_dir: Optional[str] = None) -> torch.Tensor:
"""(1, n) int64 prompt ids. Deterministic, so the ids are cached per (instruction, image tokens, processor): a
served request tokenizes its instruction twice (bucket choice, then the inputs) and robots repeat instructions."""
ids = _token_ids(str(instruction), int(num_image_tokens), processor_dir or DEFAULT_DIRS["processor"])
return torch.tensor([ids], dtype=torch.int64)
# ---------------------------------------------------------------------------------------------------------------
# Positions
# ---------------------------------------------------------------------------------------------------------------
def mrope_position_ids(input_ids: torch.Tensor, image_grid_thw: torch.Tensor) -> torch.Tensor:
"""3D mRoPE positions (3, 1, L) of Qwen2.5-VL ``get_rope_index`` for batch 1, images only, no padding."""
ids = input_ids[0].tolist()
L = len(ids)
pos = []
st = 0
for t, h, w in image_grid_thw.tolist():
ed = ids.index(IMAGE_TOKEN_ID, st)
st_idx = (pos[-1].max().item() + 1) if pos else 0
text_len = ed - st
pos.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
gh, gw = h // V.merge, w // V.merge
t_index = torch.arange(t).view(-1, 1).expand(-1, gh * gw).flatten() * 0 # images: second_per_grid_t = 0
h_index = torch.arange(gh).view(1, -1, 1).expand(t, -1, gw).flatten()
w_index = torch.arange(gw).view(1, 1, -1).expand(t, gh, -1).flatten()
pos.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx)
st = ed + t * gh * gw
if st < L:
st_idx = (pos[-1].max().item() + 1) if pos else 0
pos.append(torch.arange(L - st).view(1, -1).expand(3, -1) + st_idx)
return torch.cat(pos, dim=1).reshape(3, 1, L).to(torch.int64)
def mrope_cos_sin(position_ids: torch.Tensor, theta: float = LM.rope_theta, head_dim: int = LM.head_dim,
section: Sequence[int] = LM.mrope_section) -> Tuple[torch.Tensor, torch.Tensor]:
"""cos/sin (L, head_dim) fp32 with the mRoPE section selection already applied (t | h | w per frequency band)."""
inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.int64).float() / head_dim)) # (64,)
freqs = position_ids[:, 0, :, None].float() * inv_freq[None, None, :] # (3, L, 64)
emb = torch.cat([freqs, freqs], dim=-1) # (3, L, 128)
cos, sin = emb.cos(), emb.sin()
sec = list(section) * 2
cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(sec, dim=-1))], dim=-1)
sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(sec, dim=-1))], dim=-1)
return cos, sin
# ---------------------------------------------------------------------------------------------------------------
# Vision geometry: window order, cu_seqlens and 2D rotary of the ViT
# ---------------------------------------------------------------------------------------------------------------
def vision_window_index(grid_thw: torch.Tensor) -> Tuple[torch.Tensor, List[int]]:
"""(window_index over merged tokens, cu_window_seqlens over patches, deduplicated)."""
win = V.window // V.merge // V.patch # 4 merged tokens per window side
window_index, cu = [], [0]
base = 0
for t, h, w in grid_thw.tolist():
gh, gw = h // V.merge, w // V.merge
index = torch.arange(t * gh * gw).reshape(t, gh, gw)
pad_h = win - gh % win
pad_w = win - gw % win
nh, nw = (gh + pad_h) // win, (gw + pad_w) // win
ip = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
ip = ip.reshape(t, nh, win, nw, win).permute(0, 1, 3, 2, 4).reshape(t, nh * nw, win, win)
seqlens = (ip != -100).sum([2, 3]).reshape(-1)
ip = ip.reshape(-1)
window_index.append(ip[ip != -100] + base)
cu.extend((seqlens.cumsum(0) * V.merge ** 2 + cu[-1]).tolist())
base += t * gh * gw
cu_dedup = [cu[0]]
for c in cu[1:]:
if c != cu_dedup[-1]:
cu_dedup.append(c)
return torch.cat(window_index), cu_dedup
def vision_full_seqlens(grid_thw: torch.Tensor) -> List[int]:
cu = [0]
for t, h, w in grid_thw.tolist():
for _ in range(t):
cu.append(cu[-1] + h * w)
return cu
def vision_rotary(grid_thw: torch.Tensor) -> torch.Tensor:
"""(N, 40) rotary angles in raster-merge order (before the window permutation)."""
pos_ids = []
for t, h, w in grid_thw.tolist():
hpos = torch.arange(h).unsqueeze(1).expand(-1, w)
hpos = hpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten()
wpos = torch.arange(w).unsqueeze(0).expand(h, -1)
wpos = wpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten()
pos_ids.append(torch.stack([hpos, wpos], dim=-1).repeat(t, 1))
pos_ids = torch.cat(pos_ids, dim=0)
dim = V.head_dim // 2 # 40
inv_freq = 1.0 / (V.rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) # (20,)
max_grid = int(grid_thw[:, 1:].max())
full = torch.outer(torch.arange(max_grid, dtype=torch.float), inv_freq)
return full[pos_ids].flatten(1)
def vision_geometry(grid_thw: torch.Tensor) -> Dict[str, object]:
"""Everything input-independent the ViT needs for one image geometry (static per preset)."""
n = int(grid_thw.prod(-1).sum())
unit = V.merge ** 2
window_index, cu_window = vision_window_index(grid_thw)
rot = vision_rotary(grid_thw).reshape(n // unit, unit, -1)[window_index].reshape(n, -1)
emb = torch.cat([rot, rot], dim=-1)
patch_perm = (window_index[:, None] * unit + torch.arange(unit)[None]).reshape(-1) # windowed <- raster
return dict(n=n, window_index=window_index, patch_perm=patch_perm, reverse=torch.argsort(window_index),
cu_window=cu_window, cu_full=vision_full_seqlens(grid_thw), cos=emb.cos(), sin=emb.sin())
# ---------------------------------------------------------------------------------------------------------------
# Whole request
# ---------------------------------------------------------------------------------------------------------------
def preprocess(images: Sequence[np.ndarray], instruction: str, processor_dir: Optional[str] = None) -> Dict[str, torch.Tensor]:
"""images in model order (left_stereo, right_stereo). Returns the VLM inputs of one request (batch 1)."""
image = concat_cameras(images)
pixel_values, grid_thw = image_to_patches(image)
n_img_tok = int(grid_thw.prod()) // (V.merge ** 2)
input_ids = tokenize(instruction, n_img_tok, processor_dir)
if int((input_ids == IMAGE_TOKEN_ID).sum()) != n_img_tok:
raise RuntimeError("image token expansion mismatch")
if VISION_START_ID not in input_ids[0].tolist():
raise RuntimeError("missing <|vision_start|>")
position_ids = mrope_position_ids(input_ids, grid_thw)
return dict(image=torch.from_numpy(image), pixel_values=pixel_values, image_grid_thw=grid_thw,
input_ids=input_ids, attention_mask=torch.ones_like(input_ids), position_ids=position_ids)
# ---------------------------------------------------------------------------------------------------------------
# Static-shape bucketing (TT port): right padding
# ---------------------------------------------------------------------------------------------------------------
PROMPT_BUCKETS = (416, 448, 512)
def choose_bucket(n_tokens: int, buckets: Sequence[int] = PROMPT_BUCKETS) -> int:
for b in buckets:
if n_tokens <= b:
return b
raise ValueError(f"prompt of {n_tokens} tokens exceeds the largest bucket {buckets[-1]} (never truncated)")
def bucket_inputs(inputs: Dict[str, torch.Tensor], bucket: Optional[int] = None) -> Dict[str, torch.Tensor]:
"""Right-pad a request to a static prompt bucket.
Pad ids = <|endoftext|> (151643), pad positions continue the text positions (max + 1 + j on all 3 axes), key mask
False on the pads. Causal attention means the valid rows (and their K/V) are unchanged; the expert masks pad keys.
"""
from .config import PAD_TOKEN_ID
ids, pos = inputs["input_ids"], inputs["position_ids"]
L = ids.shape[1]
B = bucket or choose_bucket(L)
n = B - L
out = dict(inputs)
out["input_ids"] = torch.cat([ids, torch.full((1, n), PAD_TOKEN_ID, dtype=ids.dtype)], dim=1)
start = int(pos.max()) + 1
pad_pos = (torch.arange(n) + start).view(1, 1, n).expand(3, 1, n)
out["position_ids"] = torch.cat([pos, pad_pos], dim=2)
out["attention_mask"] = torch.cat([inputs["attention_mask"], torch.zeros((1, n), dtype=inputs["attention_mask"].dtype)], 1)
out["valid_len"] = torch.tensor(L)
return out