File size: 3,348 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
# SPDX-License-Identifier: Apache-2.0
"""Host-side folds the TT port relies on (each proven equal to the plain reference by tests/test_reference_host.py).

1. ``window_order_pixels``: the resized uint8 image -> raw patch rows (1512, 588) already in ViT **window order**
   (no device permute), one frame only (the processor duplicates the frame for temporal_patch_size = 2).
2. ``fold_patch_embed``: Conv3d weight (1280, 3, 2, 14, 14) -> (W', b') acting on those raw rows:
   W'[:, c] = (W[:, c, 0] + W[:, c, 1]) / (255 std_c),  b' = -sum_c (mean_c / std_c) * sum(W[:, c, 0] + W[:, c, 1]).
3. ``deinterleave_wkv``: RDT ``wkv`` (1024, 1024) with K / V interleaved per output column -> (Wk, Wv) (512, 1024) each.
4. ``lm_input_scatter_index``: where the merger rows (window order) land in the LM input (raster order after the image
   start), i.e. the inverse window permutation folded into the scatter.
"""
from __future__ import annotations

from typing import Dict, Tuple

import numpy as np
import torch
import torch.nn.functional as F

from .config import CONFIG
from .preprocess import concat_cameras, smart_resize, vision_geometry

V = CONFIG.vision
P = CONFIG.proc
E = CONFIG.expert


def resize_uint8(image: np.ndarray) -> torch.Tensor:
    x = torch.from_numpy(np.ascontiguousarray(image)).permute(2, 0, 1).contiguous()[None]
    rh, rw = smart_resize(*x.shape[-2:], factor=V.patch * V.merge)
    if (rh, rw) != tuple(x.shape[-2:]):
        x = F.interpolate(x, size=(rh, rw), mode="bicubic", antialias=True, align_corners=False)
    return x[0]  # (3, rh, rw) uint8


def window_order_pixels(images) -> Tuple[torch.Tensor, torch.Tensor]:
    """cameras (model order) -> (raw pixel rows (N, 3*14*14) float32 in window order, grid_thw)."""
    x = resize_uint8(concat_cameras(images)).float()
    _, rh, rw = x.shape
    gh, gw = rh // V.patch, rw // V.patch
    x = x.view(3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch)
    x = x.permute(1, 4, 2, 5, 0, 3, 6).reshape(gh * gw, 3 * V.patch * V.patch)  # raster-merge order, (C, 14, 14)
    grid = torch.tensor([[1, gh, gw]])
    return x[vision_geometry(grid)["patch_perm"]], grid


def fold_patch_embed(conv_w: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
    w = conv_w.double()  # (1280, 3, 2, 14, 14)
    w_eff = w.sum(dim=2)  # identical frames
    inv = 1.0 / P.rescale_factor
    mean = torch.tensor(P.image_mean, dtype=torch.float64)
    std = torch.tensor(P.image_std, dtype=torch.float64)
    w_fold = w_eff / (inv * std).view(1, 3, 1, 1)
    b_fold = -(w_eff.sum(dim=(2, 3)) * (mean / std).view(1, 3)).sum(dim=1)
    return w_fold.reshape(conv_w.shape[0], -1).float(), b_fold.float()


def deinterleave_wkv(wkv: torch.Tensor, kv_heads: int = E.kv_heads, head_dim: int = E.head_dim):
    w = wkv.view(kv_heads, head_dim, 2, wkv.shape[1])
    return w[:, :, 0].reshape(kv_heads * head_dim, -1), w[:, :, 1].reshape(kv_heads * head_dim, -1)


def lm_input_scatter_index(grid_thw: torch.Tensor, image_start: int) -> torch.Tensor:
    """row j of the merger output (window order) -> LM input row image_start + window_index[j]."""
    return image_start + vision_geometry(grid_thw)["window_index"]


def folds_summary() -> Dict[str, str]:
    return {"patch_embed": "1176 -> 588 inputs, normalization folded", "window_order": "host", "wkv": "de-interleaved"}