changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
3.35 kB
# SPDX-License-Identifier: Apache-2.0
"""Host-side folds the TT port relies on (each proven equal to the plain reference by tests/test_reference_host.py).
1. ``window_order_pixels``: the resized uint8 image -> raw patch rows (1512, 588) already in ViT **window order**
(no device permute), one frame only (the processor duplicates the frame for temporal_patch_size = 2).
2. ``fold_patch_embed``: Conv3d weight (1280, 3, 2, 14, 14) -> (W', b') acting on those raw rows:
W'[:, c] = (W[:, c, 0] + W[:, c, 1]) / (255 std_c), b' = -sum_c (mean_c / std_c) * sum(W[:, c, 0] + W[:, c, 1]).
3. ``deinterleave_wkv``: RDT ``wkv`` (1024, 1024) with K / V interleaved per output column -> (Wk, Wv) (512, 1024) each.
4. ``lm_input_scatter_index``: where the merger rows (window order) land in the LM input (raster order after the image
start), i.e. the inverse window permutation folded into the scatter.
"""
from __future__ import annotations
from typing import Dict, Tuple
import numpy as np
import torch
import torch.nn.functional as F
from .config import CONFIG
from .preprocess import concat_cameras, smart_resize, vision_geometry
V = CONFIG.vision
P = CONFIG.proc
E = CONFIG.expert
def resize_uint8(image: np.ndarray) -> torch.Tensor:
x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None]
rh, rw = smart_resize(*x.shape[-2:], factor=V.patch * V.merge)
if (rh, rw) != tuple(x.shape[-2:]):
x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False)
return x[0] # (3, rh, rw) uint8
def window_order_pixels(images) -> Tuple[torch.Tensor, torch.Tensor]:
"""cameras (model order) -> (raw pixel rows (N, 3*14*14) float32 in window order, grid_thw)."""
x = resize_uint8(concat_cameras(images)).float()
_, rh, rw = x.shape
gh, gw = rh // V.patch, rw // V.patch
x = x.view(3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch)
x = x.permute(1, 4, 2, 5, 0, 3, 6).reshape(gh * gw, 3 * V.patch * V.patch) # raster-merge order, (C, 14, 14)
grid = torch.tensor([[1, gh, gw]])
return x[vision_geometry(grid)["patch_perm"]], grid
def fold_patch_embed(conv_w: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
w = conv_w.double() # (1280, 3, 2, 14, 14)
w_eff = w.sum(dim=2) # identical frames
inv = 1.0 / P.rescale_factor
mean = torch.tensor(P.image_mean, dtype=torch.float64)
std = torch.tensor(P.image_std, dtype=torch.float64)
w_fold = w_eff / (inv * std).view(1, 3, 1, 1)
b_fold = -(w_eff.sum(dim=(2, 3)) * (mean / std).view(1, 3)).sum(dim=1)
return w_fold.reshape(conv_w.shape[0], -1).float(), b_fold.float()
def deinterleave_wkv(wkv: torch.Tensor, kv_heads: int = E.kv_heads, head_dim: int = E.head_dim):
w = wkv.view(kv_heads, head_dim, 2, wkv.shape[1])
return w[:, :, 0].reshape(kv_heads * head_dim, -1), w[:, :, 1].reshape(kv_heads * head_dim, -1)
def lm_input_scatter_index(grid_thw: torch.Tensor, image_start: int) -> torch.Tensor:
"""row j of the merger output (window order) -> LM input row image_start + window_index[j]."""
return image_start + vision_geometry(grid_thw)["window_index"]
def folds_summary() -> Dict[str, str]:
return {"patch_embed": "1176 -> 588 inputs, normalization folded", "window_order": "host", "wkv": "de-interleaved"}