Download code/tt_rdt2/reference/preprocess.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 14.2 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/preprocess.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/preprocess.py
-
curl -L -o preprocess.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/preprocess.py
14.2 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Host pre-processing of RDT2-FM, re-implemented from the pinned sources (no transformers at run time). | |
| Official path (thu-ml/RDT2 models/rdt_inferencer.py ``prepreare_data_from_vision_language_model``): | |
| images = [obs[cam] for cam in camera_names] # ["left_stereo", "right_stereo"] | |
| image = np.concatenate(images, axis=1) # (384, 768, 3) uint8, left | right | |
| text = chat_template([{user: [image, instruction]}], add_generation_prompt=False) | |
| + "<|im_start|>assistant\n<|quad_start|>" | |
| inputs = Qwen2.5-VL processor (use_fast=True, padding_side="left")(text, [image]) | |
| Processor (transformers 4.51.3 Qwen2VLImageProcessorFast + Qwen2_5_VLProcessor): | |
| smart_resize(384, 768, factor=28) -> (392, 756); torchvision bicubic+antialias resize of the uint8 CHW tensor | |
| (== torch.nn.functional.interpolate on uint8, same kernel); x = (x - 255*mean) / (255*std) in fp32; the single frame | |
| is repeated to temporal_patch_size=2; patches ordered (t, h/2, w/2, 2, 2) x (C, T, 14, 14) -> (1512, 1176); | |
| grid_thw = (1, 28, 54); "<|image_pad|>" expanded to 1512 / 4 = 378 tokens. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import os | |
| from functools import lru_cache | |
| from typing import Dict, List, Optional, Sequence, Tuple | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from .config import CONFIG, DEFAULT_DIRS, IMAGE_TOKEN_ID, VISION_START_ID | |
| V = CONFIG.vision | |
| P = CONFIG.proc | |
| LM = CONFIG.lm | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Image | |
| # --------------------------------------------------------------------------------------------------------------- | |
| def smart_resize(height: int, width: int, factor: int = 28, min_pixels: int = P.min_pixels, | |
| max_pixels: int = P.max_pixels) -> Tuple[int, int]: | |
| if height < factor or width < factor: | |
| raise ValueError(f"height:{height} and width:{width} must be larger than factor:{factor}") | |
| if max(height, width) / min(height, width) > 200: | |
| raise ValueError("absolute aspect ratio must be smaller than 200") | |
| h_bar = round(height / factor) * factor | |
| w_bar = round(width / factor) * factor | |
| if h_bar * w_bar > max_pixels: | |
| beta = math.sqrt((height * width) / max_pixels) | |
| h_bar = math.floor(height / beta / factor) * factor | |
| w_bar = math.floor(width / beta / factor) * factor | |
| elif h_bar * w_bar < min_pixels: | |
| beta = math.sqrt(min_pixels / (height * width)) | |
| h_bar = math.ceil(height * beta / factor) * factor | |
| w_bar = math.ceil(width * beta / factor) * factor | |
| return h_bar, w_bar | |
| def concat_cameras(images: Sequence[np.ndarray]) -> np.ndarray: | |
| """Cameras in model order (left_stereo, right_stereo), each (H, W, 3) uint8 -> (H, sum W, 3).""" | |
| for im in images: | |
| if im.dtype != np.uint8 or im.ndim != 3 or im.shape[2] != 3: | |
| raise ValueError(f"expected HxWx3 uint8 images, got {im.dtype} {im.shape}") | |
| return np.ascontiguousarray(np.concatenate(images, axis=1)) | |
| def image_to_patches(image: np.ndarray) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """(H, W, 3) uint8 -> pixel_values (N, 1176) fp32, grid_thw (1, 3) int64 (exactly the fast processor's output).""" | |
| x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None] # (1, 3, H, W) uint8 | |
| h, w = x.shape[-2:] | |
| rh, rw = smart_resize(h, w, factor=V.patch * V.merge) | |
| if (rh, rw) != (h, w): | |
| x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False) | |
| inv = 1.0 / P.rescale_factor | |
| mean = torch.tensor(P.image_mean) * inv | |
| std = torch.tensor(P.image_std) * inv | |
| x = (x.to(torch.float32) - mean.view(1, 3, 1, 1)) / std.view(1, 3, 1, 1) | |
| x = x.repeat(V.temporal_patch, 1, 1, 1) # (T=2, 3, H, W): single frame repeated | |
| gt, gh, gw = 1, rh // V.patch, rw // V.patch | |
| x = x.view(gt, V.temporal_patch, 3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch) | |
| x = x.permute(0, 3, 6, 4, 7, 2, 1, 5, 8) | |
| pv = x.reshape(gt * gh * gw, 3 * V.temporal_patch * V.patch * V.patch) | |
| return pv.contiguous(), torch.tensor([[gt, gh, gw]], dtype=torch.int64) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Prompt / tokens | |
| # --------------------------------------------------------------------------------------------------------------- | |
| def build_prompt(instruction: str, num_image_tokens: int) -> str: | |
| """Official text (chat template with one user turn [image, text], no generation prompt, + the guidance suffix), | |
| with "<|image_pad|>" already expanded the way Qwen2_5_VLProcessor does.""" | |
| return ("<|im_start|>system\n" + P.system_prompt + "<|im_end|>\n<|im_start|>user\n<|vision_start|>" | |
| + "<|image_pad|>" * num_image_tokens + "<|vision_end|>" + instruction + "<|im_end|>\n" + P.suffix) | |
| def _tokenizer(processor_dir: str): | |
| from tokenizers import Tokenizer | |
| return Tokenizer.from_file(os.path.join(processor_dir, "tokenizer.json")) | |
| def _token_ids(instruction: str, num_image_tokens: int, processor_dir: str) -> Tuple[int, ...]: | |
| tok = _tokenizer(processor_dir) | |
| return tuple(tok.encode(build_prompt(instruction, num_image_tokens), add_special_tokens=True).ids) | |
| def tokenize(instruction: str, num_image_tokens: int, processor_dir: Optional[str] = None) -> torch.Tensor: | |
| """(1, n) int64 prompt ids. Deterministic, so the ids are cached per (instruction, image tokens, processor): a | |
| served request tokenizes its instruction twice (bucket choice, then the inputs) and robots repeat instructions.""" | |
| ids = _token_ids(str(instruction), int(num_image_tokens), processor_dir or DEFAULT_DIRS["processor"]) | |
| return torch.tensor([ids], dtype=torch.int64) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Positions | |
| # --------------------------------------------------------------------------------------------------------------- | |
| def mrope_position_ids(input_ids: torch.Tensor, image_grid_thw: torch.Tensor) -> torch.Tensor: | |
| """3D mRoPE positions (3, 1, L) of Qwen2.5-VL ``get_rope_index`` for batch 1, images only, no padding.""" | |
| ids = input_ids[0].tolist() | |
| L = len(ids) | |
| pos = [] | |
| st = 0 | |
| for t, h, w in image_grid_thw.tolist(): | |
| ed = ids.index(IMAGE_TOKEN_ID, st) | |
| st_idx = (pos[-1].max().item() + 1) if pos else 0 | |
| text_len = ed - st | |
| pos.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) | |
| gh, gw = h // V.merge, w // V.merge | |
| t_index = torch.arange(t).view(-1, 1).expand(-1, gh * gw).flatten() * 0 # images: second_per_grid_t = 0 | |
| h_index = torch.arange(gh).view(1, -1, 1).expand(t, -1, gw).flatten() | |
| w_index = torch.arange(gw).view(1, 1, -1).expand(t, gh, -1).flatten() | |
| pos.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) | |
| st = ed + t * gh * gw | |
| if st < L: | |
| st_idx = (pos[-1].max().item() + 1) if pos else 0 | |
| pos.append(torch.arange(L - st).view(1, -1).expand(3, -1) + st_idx) | |
| return torch.cat(pos, dim=1).reshape(3, 1, L).to(torch.int64) | |
| def mrope_cos_sin(position_ids: torch.Tensor, theta: float = LM.rope_theta, head_dim: int = LM.head_dim, | |
| section: Sequence[int] = LM.mrope_section) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """cos/sin (L, head_dim) fp32 with the mRoPE section selection already applied (t | h | w per frequency band).""" | |
| inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.int64).float() / head_dim)) # (64,) | |
| freqs = position_ids[:, 0, :, None].float() * inv_freq[None, None, :] # (3, L, 64) | |
| emb = torch.cat([freqs, freqs], dim=-1) # (3, L, 128) | |
| cos, sin = emb.cos(), emb.sin() | |
| sec = list(section) * 2 | |
| cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(sec, dim=-1))], dim=-1) | |
| sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(sec, dim=-1))], dim=-1) | |
| return cos, sin | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Vision geometry: window order, cu_seqlens and 2D rotary of the ViT | |
| # --------------------------------------------------------------------------------------------------------------- | |
| def vision_window_index(grid_thw: torch.Tensor) -> Tuple[torch.Tensor, List[int]]: | |
| """(window_index over merged tokens, cu_window_seqlens over patches, deduplicated).""" | |
| win = V.window // V.merge // V.patch # 4 merged tokens per window side | |
| window_index, cu = [], [0] | |
| base = 0 | |
| for t, h, w in grid_thw.tolist(): | |
| gh, gw = h // V.merge, w // V.merge | |
| index = torch.arange(t * gh * gw).reshape(t, gh, gw) | |
| pad_h = win - gh % win | |
| pad_w = win - gw % win | |
| nh, nw = (gh + pad_h) // win, (gw + pad_w) // win | |
| ip = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100) | |
| ip = ip.reshape(t, nh, win, nw, win).permute(0, 1, 3, 2, 4).reshape(t, nh * nw, win, win) | |
| seqlens = (ip != -100).sum([2, 3]).reshape(-1) | |
| ip = ip.reshape(-1) | |
| window_index.append(ip[ip != -100] + base) | |
| cu.extend((seqlens.cumsum(0) * V.merge ** 2 + cu[-1]).tolist()) | |
| base += t * gh * gw | |
| cu_dedup = [cu[0]] | |
| for c in cu[1:]: | |
| if c != cu_dedup[-1]: | |
| cu_dedup.append(c) | |
| return torch.cat(window_index), cu_dedup | |
| def vision_full_seqlens(grid_thw: torch.Tensor) -> List[int]: | |
| cu = [0] | |
| for t, h, w in grid_thw.tolist(): | |
| for _ in range(t): | |
| cu.append(cu[-1] + h * w) | |
| return cu | |
| def vision_rotary(grid_thw: torch.Tensor) -> torch.Tensor: | |
| """(N, 40) rotary angles in raster-merge order (before the window permutation).""" | |
| pos_ids = [] | |
| for t, h, w in grid_thw.tolist(): | |
| hpos = torch.arange(h).unsqueeze(1).expand(-1, w) | |
| hpos = hpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten() | |
| wpos = torch.arange(w).unsqueeze(0).expand(h, -1) | |
| wpos = wpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten() | |
| pos_ids.append(torch.stack([hpos, wpos], dim=-1).repeat(t, 1)) | |
| pos_ids = torch.cat(pos_ids, dim=0) | |
| dim = V.head_dim // 2 # 40 | |
| inv_freq = 1.0 / (V.rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) # (20,) | |
| max_grid = int(grid_thw[:, 1:].max()) | |
| full = torch.outer(torch.arange(max_grid, dtype=torch.float), inv_freq) | |
| return full[pos_ids].flatten(1) | |
| def vision_geometry(grid_thw: torch.Tensor) -> Dict[str, object]: | |
| """Everything input-independent the ViT needs for one image geometry (static per preset).""" | |
| n = int(grid_thw.prod(-1).sum()) | |
| unit = V.merge ** 2 | |
| window_index, cu_window = vision_window_index(grid_thw) | |
| rot = vision_rotary(grid_thw).reshape(n // unit, unit, -1)[window_index].reshape(n, -1) | |
| emb = torch.cat([rot, rot], dim=-1) | |
| patch_perm = (window_index[:, None] * unit + torch.arange(unit)[None]).reshape(-1) # windowed <- raster | |
| return dict(n=n, window_index=window_index, patch_perm=patch_perm, reverse=torch.argsort(window_index), | |
| cu_window=cu_window, cu_full=vision_full_seqlens(grid_thw), cos=emb.cos(), sin=emb.sin()) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Whole request | |
| # --------------------------------------------------------------------------------------------------------------- | |
| def preprocess(images: Sequence[np.ndarray], instruction: str, processor_dir: Optional[str] = None) -> Dict[str, torch.Tensor]: | |
| """images in model order (left_stereo, right_stereo). Returns the VLM inputs of one request (batch 1).""" | |
| image = concat_cameras(images) | |
| pixel_values, grid_thw = image_to_patches(image) | |
| n_img_tok = int(grid_thw.prod()) // (V.merge ** 2) | |
| input_ids = tokenize(instruction, n_img_tok, processor_dir) | |
| if int((input_ids == IMAGE_TOKEN_ID).sum()) != n_img_tok: | |
| raise RuntimeError("image token expansion mismatch") | |
| if VISION_START_ID not in input_ids[0].tolist(): | |
| raise RuntimeError("missing <|vision_start|>") | |
| position_ids = mrope_position_ids(input_ids, grid_thw) | |
| return dict(image=torch.from_numpy(image), pixel_values=pixel_values, image_grid_thw=grid_thw, | |
| input_ids=input_ids, attention_mask=torch.ones_like(input_ids), position_ids=position_ids) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Static-shape bucketing (TT port): right padding | |
| # --------------------------------------------------------------------------------------------------------------- | |
| PROMPT_BUCKETS = (416, 448, 512) | |
| def choose_bucket(n_tokens: int, buckets: Sequence[int] = PROMPT_BUCKETS) -> int: | |
| for b in buckets: | |
| if n_tokens <= b: | |
| return b | |
| raise ValueError(f"prompt of {n_tokens} tokens exceeds the largest bucket {buckets[-1]} (never truncated)") | |
| def bucket_inputs(inputs: Dict[str, torch.Tensor], bucket: Optional[int] = None) -> Dict[str, torch.Tensor]: | |
| """Right-pad a request to a static prompt bucket. | |
| Pad ids = <|endoftext|> (151643), pad positions continue the text positions (max + 1 + j on all 3 axes), key mask | |
| False on the pads. Causal attention means the valid rows (and their K/V) are unchanged; the expert masks pad keys. | |
| """ | |
| from .config import PAD_TOKEN_ID | |
| ids, pos = inputs["input_ids"], inputs["position_ids"] | |
| L = ids.shape[1] | |
| B = bucket or choose_bucket(L) | |
| n = B - L | |
| out = dict(inputs) | |
| out["input_ids"] = torch.cat([ids, torch.full((1, n), PAD_TOKEN_ID, dtype=ids.dtype)], dim=1) | |
| start = int(pos.max()) + 1 | |
| pad_pos = (torch.arange(n) + start).view(1, 1, n).expand(3, 1, n) | |
| out["position_ids"] = torch.cat([pos, pad_pos], dim=2) | |
| out["attention_mask"] = torch.cat([inputs["attention_mask"], torch.zeros((1, n), dtype=inputs["attention_mask"].dtype)], 1) | |
| out["valid_len"] = torch.tensor(L) | |
| return out | |