File size: 14,224 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
# SPDX-License-Identifier: Apache-2.0
"""Host pre-processing of RDT2-FM, re-implemented from the pinned sources (no transformers at run time).

Official path (thu-ml/RDT2 models/rdt_inferencer.py ``prepreare_data_from_vision_language_model``):
  images = [obs[cam] for cam in camera_names]           # ["left_stereo", "right_stereo"]
  image  = np.concatenate(images, axis=1)               # (384, 768, 3) uint8, left | right
  text   = chat_template([{user: [image, instruction]}], add_generation_prompt=False)
           + "<|im_start|>assistant\n<|quad_start|>"
  inputs = Qwen2.5-VL processor (use_fast=True, padding_side="left")(text, [image])

Processor (transformers 4.51.3 Qwen2VLImageProcessorFast + Qwen2_5_VLProcessor):
  smart_resize(384, 768, factor=28) -> (392, 756); torchvision bicubic+antialias resize of the uint8 CHW tensor
  (== torch.nn.functional.interpolate on uint8, same kernel); x = (x - 255*mean) / (255*std) in fp32; the single frame
  is repeated to temporal_patch_size=2; patches ordered (t, h/2, w/2, 2, 2) x (C, T, 14, 14) -> (1512, 1176);
  grid_thw = (1, 28, 54); "<|image_pad|>" expanded to 1512 / 4 = 378 tokens.
"""
from __future__ import annotations

import math
import os
from functools import lru_cache
from typing import Dict, List, Optional, Sequence, Tuple

import numpy as np
import torch
import torch.nn.functional as F

from .config import CONFIG, DEFAULT_DIRS, IMAGE_TOKEN_ID, VISION_START_ID

V = CONFIG.vision
P = CONFIG.proc
LM = CONFIG.lm


# ---------------------------------------------------------------------------------------------------------------
# Image
# ---------------------------------------------------------------------------------------------------------------
def smart_resize(height: int, width: int, factor: int = 28, min_pixels: int = P.min_pixels,
                 max_pixels: int = P.max_pixels) -> Tuple[int, int]:
    if height < factor or width < factor:
        raise ValueError(f"height:{height} and width:{width} must be larger than factor:{factor}")
    if max(height, width) / min(height, width) > 200:
        raise ValueError("absolute aspect ratio must be smaller than 200")
    h_bar = round(height / factor) * factor
    w_bar = round(width / factor) * factor
    if h_bar * w_bar > max_pixels:
        beta = math.sqrt((height * width) / max_pixels)
        h_bar = math.floor(height / beta / factor) * factor
        w_bar = math.floor(width / beta / factor) * factor
    elif h_bar * w_bar < min_pixels:
        beta = math.sqrt(min_pixels / (height * width))
        h_bar = math.ceil(height * beta / factor) * factor
        w_bar = math.ceil(width * beta / factor) * factor
    return h_bar, w_bar


def concat_cameras(images: Sequence[np.ndarray]) -> np.ndarray:
    """Cameras in model order (left_stereo, right_stereo), each (H, W, 3) uint8 -> (H, sum W, 3)."""
    for im in images:
        if im.dtype != np.uint8 or im.ndim != 3 or im.shape[2] != 3:
            raise ValueError(f"expected HxWx3 uint8 images, got {im.dtype} {im.shape}")
    return np.ascontiguousarray(np.concatenate(images, axis=1))


def image_to_patches(image: np.ndarray) -> Tuple[torch.Tensor, torch.Tensor]:
    """(H, W, 3) uint8 -> pixel_values (N, 1176) fp32, grid_thw (1, 3) int64 (exactly the fast processor's output)."""
    x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None]  # (1, 3, H, W) uint8
    h, w = x.shape[-2:]
    rh, rw = smart_resize(h, w, factor=V.patch * V.merge)
    if (rh, rw) != (h, w):
        x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False)
    inv = 1.0 / P.rescale_factor
    mean = torch.tensor(P.image_mean) * inv
    std = torch.tensor(P.image_std) * inv
    x = (x.to(torch.float32) - mean.view(1, 3, 1, 1)) / std.view(1, 3, 1, 1)
    x = x.repeat(V.temporal_patch, 1, 1, 1)  # (T=2, 3, H, W): single frame repeated
    gt, gh, gw = 1, rh // V.patch, rw // V.patch
    x = x.view(gt, V.temporal_patch, 3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch)
    x = x.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)
    pv = x.reshape(gt * gh * gw, 3 * V.temporal_patch * V.patch * V.patch)
    return pv.contiguous(), torch.tensor([[gt, gh, gw]], dtype=torch.int64)


# ---------------------------------------------------------------------------------------------------------------
# Prompt / tokens
# ---------------------------------------------------------------------------------------------------------------
def build_prompt(instruction: str, num_image_tokens: int) -> str:
    """Official text (chat template with one user turn [image, text], no generation prompt, + the guidance suffix),
    with "<|image_pad|>" already expanded the way Qwen2_5_VLProcessor does."""
    return ("<|im_start|>system\n" + P.system_prompt + "<|im_end|>\n<|im_start|>user\n<|vision_start|>"
            + "<|image_pad|>" * num_image_tokens + "<|vision_end|>" + instruction + "<|im_end|>\n" + P.suffix)


@lru_cache(maxsize=2)
def _tokenizer(processor_dir: str):
    from tokenizers import Tokenizer

    return Tokenizer.from_file(os.path.join(processor_dir, "tokenizer.json"))


@lru_cache(maxsize=256)
def _token_ids(instruction: str, num_image_tokens: int, processor_dir: str) -> Tuple[int, ...]:
    tok = _tokenizer(processor_dir)
    return tuple(tok.encode(build_prompt(instruction, num_image_tokens), add_special_tokens=True).ids)


def tokenize(instruction: str, num_image_tokens: int, processor_dir: Optional[str] = None) -> torch.Tensor:
    """(1, n) int64 prompt ids. Deterministic, so the ids are cached per (instruction, image tokens, processor): a
    served request tokenizes its instruction twice (bucket choice, then the inputs) and robots repeat instructions."""
    ids = _token_ids(str(instruction), int(num_image_tokens), processor_dir or DEFAULT_DIRS["processor"])
    return torch.tensor([ids], dtype=torch.int64)


# ---------------------------------------------------------------------------------------------------------------
# Positions
# ---------------------------------------------------------------------------------------------------------------
def mrope_position_ids(input_ids: torch.Tensor, image_grid_thw: torch.Tensor) -> torch.Tensor:
    """3D mRoPE positions (3, 1, L) of Qwen2.5-VL ``get_rope_index`` for batch 1, images only, no padding."""
    ids = input_ids[0].tolist()
    L = len(ids)
    pos = []
    st = 0
    for t, h, w in image_grid_thw.tolist():
        ed = ids.index(IMAGE_TOKEN_ID, st)
        st_idx = (pos[-1].max().item() + 1) if pos else 0
        text_len = ed - st
        pos.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
        gh, gw = h // V.merge, w // V.merge
        t_index = torch.arange(t).view(-1, 1).expand(-1, gh * gw).flatten() * 0  # images: second_per_grid_t = 0
        h_index = torch.arange(gh).view(1, -1, 1).expand(t, -1, gw).flatten()
        w_index = torch.arange(gw).view(1, 1, -1).expand(t, gh, -1).flatten()
        pos.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx)
        st = ed + t * gh * gw
    if st < L:
        st_idx = (pos[-1].max().item() + 1) if pos else 0
        pos.append(torch.arange(L - st).view(1, -1).expand(3, -1) + st_idx)
    return torch.cat(pos, dim=1).reshape(3, 1, L).to(torch.int64)


def mrope_cos_sin(position_ids: torch.Tensor, theta: float = LM.rope_theta, head_dim: int = LM.head_dim,
                  section: Sequence[int] = LM.mrope_section) -> Tuple[torch.Tensor, torch.Tensor]:
    """cos/sin (L, head_dim) fp32 with the mRoPE section selection already applied (t | h | w per frequency band)."""
    inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.int64).float() / head_dim))  # (64,)
    freqs = position_ids[:, 0, :, None].float() * inv_freq[None, None, :]  # (3, L, 64)
    emb = torch.cat([freqs, freqs], dim=-1)  # (3, L, 128)
    cos, sin = emb.cos(), emb.sin()
    sec = list(section) * 2
    cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(sec, dim=-1))], dim=-1)
    sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(sec, dim=-1))], dim=-1)
    return cos, sin


# ---------------------------------------------------------------------------------------------------------------
# Vision geometry: window order, cu_seqlens and 2D rotary of the ViT
# ---------------------------------------------------------------------------------------------------------------
def vision_window_index(grid_thw: torch.Tensor) -> Tuple[torch.Tensor, List[int]]:
    """(window_index over merged tokens, cu_window_seqlens over patches, deduplicated)."""
    win = V.window // V.merge // V.patch  # 4 merged tokens per window side
    window_index, cu = [], [0]
    base = 0
    for t, h, w in grid_thw.tolist():
        gh, gw = h // V.merge, w // V.merge
        index = torch.arange(t * gh * gw).reshape(t, gh, gw)
        pad_h = win - gh % win
        pad_w = win - gw % win
        nh, nw = (gh + pad_h) // win, (gw + pad_w) // win
        ip = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
        ip = ip.reshape(t, nh, win, nw, win).permute(0, 1, 3, 2, 4).reshape(t, nh * nw, win, win)
        seqlens = (ip != -100).sum([2, 3]).reshape(-1)
        ip = ip.reshape(-1)
        window_index.append(ip[ip != -100] + base)
        cu.extend((seqlens.cumsum(0) * V.merge ** 2 + cu[-1]).tolist())
        base += t * gh * gw
    cu_dedup = [cu[0]]
    for c in cu[1:]:
        if c != cu_dedup[-1]:
            cu_dedup.append(c)
    return torch.cat(window_index), cu_dedup


def vision_full_seqlens(grid_thw: torch.Tensor) -> List[int]:
    cu = [0]
    for t, h, w in grid_thw.tolist():
        for _ in range(t):
            cu.append(cu[-1] + h * w)
    return cu


def vision_rotary(grid_thw: torch.Tensor) -> torch.Tensor:
    """(N, 40) rotary angles in raster-merge order (before the window permutation)."""
    pos_ids = []
    for t, h, w in grid_thw.tolist():
        hpos = torch.arange(h).unsqueeze(1).expand(-1, w)
        hpos = hpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten()
        wpos = torch.arange(w).unsqueeze(0).expand(h, -1)
        wpos = wpos.reshape(h // V.merge, V.merge, w // V.merge, V.merge).permute(0, 2, 1, 3).flatten()
        pos_ids.append(torch.stack([hpos, wpos], dim=-1).repeat(t, 1))
    pos_ids = torch.cat(pos_ids, dim=0)
    dim = V.head_dim // 2  # 40
    inv_freq = 1.0 / (V.rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))  # (20,)
    max_grid = int(grid_thw[:, 1:].max())
    full = torch.outer(torch.arange(max_grid, dtype=torch.float), inv_freq)
    return full[pos_ids].flatten(1)


def vision_geometry(grid_thw: torch.Tensor) -> Dict[str, object]:
    """Everything input-independent the ViT needs for one image geometry (static per preset)."""
    n = int(grid_thw.prod(-1).sum())
    unit = V.merge ** 2
    window_index, cu_window = vision_window_index(grid_thw)
    rot = vision_rotary(grid_thw).reshape(n // unit, unit, -1)[window_index].reshape(n, -1)
    emb = torch.cat([rot, rot], dim=-1)
    patch_perm = (window_index[:, None] * unit + torch.arange(unit)[None]).reshape(-1)  # windowed <- raster
    return dict(n=n, window_index=window_index, patch_perm=patch_perm, reverse=torch.argsort(window_index),
                cu_window=cu_window, cu_full=vision_full_seqlens(grid_thw), cos=emb.cos(), sin=emb.sin())


# ---------------------------------------------------------------------------------------------------------------
# Whole request
# ---------------------------------------------------------------------------------------------------------------
def preprocess(images: Sequence[np.ndarray], instruction: str, processor_dir: Optional[str] = None) -> Dict[str, torch.Tensor]:
    """images in model order (left_stereo, right_stereo). Returns the VLM inputs of one request (batch 1)."""
    image = concat_cameras(images)
    pixel_values, grid_thw = image_to_patches(image)
    n_img_tok = int(grid_thw.prod()) // (V.merge ** 2)
    input_ids = tokenize(instruction, n_img_tok, processor_dir)
    if int((input_ids == IMAGE_TOKEN_ID).sum()) != n_img_tok:
        raise RuntimeError("image token expansion mismatch")
    if VISION_START_ID not in input_ids[0].tolist():
        raise RuntimeError("missing <|vision_start|>")
    position_ids = mrope_position_ids(input_ids, grid_thw)
    return dict(image=torch.from_numpy(image), pixel_values=pixel_values, image_grid_thw=grid_thw,
                input_ids=input_ids, attention_mask=torch.ones_like(input_ids), position_ids=position_ids)


# ---------------------------------------------------------------------------------------------------------------
# Static-shape bucketing (TT port): right padding
# ---------------------------------------------------------------------------------------------------------------
PROMPT_BUCKETS = (416, 448, 512)


def choose_bucket(n_tokens: int, buckets: Sequence[int] = PROMPT_BUCKETS) -> int:
    for b in buckets:
        if n_tokens <= b:
            return b
    raise ValueError(f"prompt of {n_tokens} tokens exceeds the largest bucket {buckets[-1]} (never truncated)")


def bucket_inputs(inputs: Dict[str, torch.Tensor], bucket: Optional[int] = None) -> Dict[str, torch.Tensor]:
    """Right-pad a request to a static prompt bucket.

    Pad ids = <|endoftext|> (151643), pad positions continue the text positions (max + 1 + j on all 3 axes), key mask
    False on the pads. Causal attention means the valid rows (and their K/V) are unchanged; the expert masks pad keys.
    """
    from .config import PAD_TOKEN_ID

    ids, pos = inputs["input_ids"], inputs["position_ids"]
    L = ids.shape[1]
    B = bucket or choose_bucket(L)
    n = B - L
    out = dict(inputs)
    out["input_ids"] = torch.cat([ids, torch.full((1, n), PAD_TOKEN_ID, dtype=ids.dtype)], dim=1)
    start = int(pos.max()) + 1
    pad_pos = (torch.arange(n) + start).view(1, 1, n).expand(3, 1, n)
    out["position_ids"] = torch.cat([pos, pad_pos], dim=2)
    out["attention_mask"] = torch.cat([inputs["attention_mask"], torch.zeros((1, n), dtype=inputs["attention_mask"].dtype)], 1)
    out["valid_len"] = torch.tensor(L)
    return out