Download code/tt_rdt2/reference/host_fold.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 3.35 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/host_fold.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/host_fold.py
-
curl -L -o host_fold.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/host_fold.py
3.35 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Host-side folds the TT port relies on (each proven equal to the plain reference by tests/test_reference_host.py). | |
| 1. ``window_order_pixels``: the resized uint8 image -> raw patch rows (1512, 588) already in ViT **window order** | |
| (no device permute), one frame only (the processor duplicates the frame for temporal_patch_size = 2). | |
| 2. ``fold_patch_embed``: Conv3d weight (1280, 3, 2, 14, 14) -> (W', b') acting on those raw rows: | |
| W'[:, c] = (W[:, c, 0] + W[:, c, 1]) / (255 std_c), b' = -sum_c (mean_c / std_c) * sum(W[:, c, 0] + W[:, c, 1]). | |
| 3. ``deinterleave_wkv``: RDT ``wkv`` (1024, 1024) with K / V interleaved per output column -> (Wk, Wv) (512, 1024) each. | |
| 4. ``lm_input_scatter_index``: where the merger rows (window order) land in the LM input (raster order after the image | |
| start), i.e. the inverse window permutation folded into the scatter. | |
| """ | |
| from __future__ import annotations | |
| from typing import Dict, Tuple | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from .config import CONFIG | |
| from .preprocess import concat_cameras, smart_resize, vision_geometry | |
| V = CONFIG.vision | |
| P = CONFIG.proc | |
| E = CONFIG.expert | |
| def resize_uint8(image: np.ndarray) -> torch.Tensor: | |
| x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None] | |
| rh, rw = smart_resize(*x.shape[-2:], factor=V.patch * V.merge) | |
| if (rh, rw) != tuple(x.shape[-2:]): | |
| x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False) | |
| return x[0] # (3, rh, rw) uint8 | |
| def window_order_pixels(images) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """cameras (model order) -> (raw pixel rows (N, 3*14*14) float32 in window order, grid_thw).""" | |
| x = resize_uint8(concat_cameras(images)).float() | |
| _, rh, rw = x.shape | |
| gh, gw = rh // V.patch, rw // V.patch | |
| x = x.view(3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch) | |
| x = x.permute(1, 4, 2, 5, 0, 3, 6).reshape(gh * gw, 3 * V.patch * V.patch) # raster-merge order, (C, 14, 14) | |
| grid = torch.tensor([[1, gh, gw]]) | |
| return x[vision_geometry(grid)["patch_perm"]], grid | |
| def fold_patch_embed(conv_w: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: | |
| w = conv_w.double() # (1280, 3, 2, 14, 14) | |
| w_eff = w.sum(dim=2) # identical frames | |
| inv = 1.0 / P.rescale_factor | |
| mean = torch.tensor(P.image_mean, dtype=torch.float64) | |
| std = torch.tensor(P.image_std, dtype=torch.float64) | |
| w_fold = w_eff / (inv * std).view(1, 3, 1, 1) | |
| b_fold = -(w_eff.sum(dim=(2, 3)) * (mean / std).view(1, 3)).sum(dim=1) | |
| return w_fold.reshape(conv_w.shape[0], -1).float(), b_fold.float() | |
| def deinterleave_wkv(wkv: torch.Tensor, kv_heads: int = E.kv_heads, head_dim: int = E.head_dim): | |
| w = wkv.view(kv_heads, head_dim, 2, wkv.shape[1]) | |
| return w[:, :, 0].reshape(kv_heads * head_dim, -1), w[:, :, 1].reshape(kv_heads * head_dim, -1) | |
| def lm_input_scatter_index(grid_thw: torch.Tensor, image_start: int) -> torch.Tensor: | |
| """row j of the merger output (window order) -> LM input row image_start + window_index[j].""" | |
| return image_start + vision_geometry(grid_thw)["window_index"] | |
| def folds_summary() -> Dict[str, str]: | |
| return {"patch_embed": "1176 -> 588 inputs, normalization folded", "window_order": "host", "wkv": "de-interleaved"} | |