# SPDX-License-Identifier: Apache-2.0 """Host-side folds the TT port relies on (each proven equal to the plain reference by tests/test_reference_host.py). 1. ``window_order_pixels``: the resized uint8 image -> raw patch rows (1512, 588) already in ViT **window order** (no device permute), one frame only (the processor duplicates the frame for temporal_patch_size = 2). 2. ``fold_patch_embed``: Conv3d weight (1280, 3, 2, 14, 14) -> (W', b') acting on those raw rows: W'[:, c] = (W[:, c, 0] + W[:, c, 1]) / (255 std_c), b' = -sum_c (mean_c / std_c) * sum(W[:, c, 0] + W[:, c, 1]). 3. ``deinterleave_wkv``: RDT ``wkv`` (1024, 1024) with K / V interleaved per output column -> (Wk, Wv) (512, 1024) each. 4. ``lm_input_scatter_index``: where the merger rows (window order) land in the LM input (raster order after the image start), i.e. the inverse window permutation folded into the scatter. """ from __future__ import annotations from typing import Dict, Tuple import numpy as np import torch import torch.nn.functional as F from .config import CONFIG from .preprocess import concat_cameras, smart_resize, vision_geometry V = CONFIG.vision P = CONFIG.proc E = CONFIG.expert def resize_uint8(image: np.ndarray) -> torch.Tensor: x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None] rh, rw = smart_resize(*x.shape[-2:], factor=V.patch * V.merge) if (rh, rw) != tuple(x.shape[-2:]): x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False) return x[0] # (3, rh, rw) uint8 def window_order_pixels(images) -> Tuple[torch.Tensor, torch.Tensor]: """cameras (model order) -> (raw pixel rows (N, 3*14*14) float32 in window order, grid_thw).""" x = resize_uint8(concat_cameras(images)).float() _, rh, rw = x.shape gh, gw = rh // V.patch, rw // V.patch x = x.view(3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch) x = x.permute(1, 4, 2, 5, 0, 3, 6).reshape(gh * gw, 3 * V.patch * V.patch) # raster-merge order, (C, 14, 14) grid = torch.tensor([[1, gh, gw]]) return x[vision_geometry(grid)["patch_perm"]], grid def fold_patch_embed(conv_w: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: w = conv_w.double() # (1280, 3, 2, 14, 14) w_eff = w.sum(dim=2) # identical frames inv = 1.0 / P.rescale_factor mean = torch.tensor(P.image_mean, dtype=torch.float64) std = torch.tensor(P.image_std, dtype=torch.float64) w_fold = w_eff / (inv * std).view(1, 3, 1, 1) b_fold = -(w_eff.sum(dim=(2, 3)) * (mean / std).view(1, 3)).sum(dim=1) return w_fold.reshape(conv_w.shape[0], -1).float(), b_fold.float() def deinterleave_wkv(wkv: torch.Tensor, kv_heads: int = E.kv_heads, head_dim: int = E.head_dim): w = wkv.view(kv_heads, head_dim, 2, wkv.shape[1]) return w[:, :, 0].reshape(kv_heads * head_dim, -1), w[:, :, 1].reshape(kv_heads * head_dim, -1) def lm_input_scatter_index(grid_thw: torch.Tensor, image_start: int) -> torch.Tensor: """row j of the merger output (window order) -> LM input row image_start + window_index[j].""" return image_start + vision_geometry(grid_thw)["window_index"] def folds_summary() -> Dict[str, str]: return {"patch_embed": "1176 -> 588 inputs, normalization folded", "window_order": "host", "wkv": "de-interleaved"}