File size: 3,177 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# SPDX-License-Identifier: Apache-2.0
"""Weight loading straight from the pinned downloads (no official code, no transformers).

* VLM (RDT2-VQ, Qwen2.5-VL-7B): 4 bf16 safetensors shards + model.safetensors.index.json. Only the tensors a stage
  needs are read (``safe_open`` per key); they stay bf16 in RAM and are upcast to fp32 on use (exact).
* Action expert (RDT2-FM): ``pytorch_model.bin`` (bf16 pickle) via ``torch.load(weights_only=True)``.
* Action normalizer: ``umi_normalizer_wo_downsample_indentity_rot.pt`` (pickled ``nn.ParameterDict``) via
  ``torch.load(weights_only=True)`` with ``ParameterDict`` allow-listed; only ``action.scale/offset`` are used.
"""
from __future__ import annotations

import json
import os
from typing import Dict, Iterable, Optional

import torch

from .config import DEFAULT_DIRS, NORMALIZER_FILE


class SafetensorsIndex:
    """Lazy reader of a sharded safetensors checkpoint (keys -> bf16 tensors, cached)."""

    def __init__(self, root: str):
        from safetensors import safe_open  # local import: keep module import light

        self._safe_open = safe_open
        self.root = root
        with open(os.path.join(root, "model.safetensors.index.json")) as f:
            self.weight_map: Dict[str, str] = json.load(f)["weight_map"]
        self._cache: Dict[str, torch.Tensor] = {}

    def keys(self):
        return self.weight_map.keys()

    def load(self, names: Iterable[str]) -> Dict[str, torch.Tensor]:
        by_file: Dict[str, list] = {}
        for n in names:
            if n not in self._cache:
                by_file.setdefault(self.weight_map[n], []).append(n)
        for fn, ns in by_file.items():
            with self._safe_open(os.path.join(self.root, fn), framework="pt", device="cpu") as f:
                for n in ns:
                    self._cache[n] = f.get_tensor(n)
        return {n: self._cache[n] for n in names}

    def load_prefix(self, prefix: str) -> Dict[str, torch.Tensor]:
        return self.load([k for k in self.weight_map if k.startswith(prefix)])

    def get(self, name: str) -> torch.Tensor:
        return self.load([name])[name]

    def drop(self, prefix: str = "") -> None:
        for k in [k for k in self._cache if k.startswith(prefix)]:
            del self._cache[k]


def load_vlm_index(root: Optional[str] = None) -> SafetensorsIndex:
    return SafetensorsIndex(root or DEFAULT_DIRS["vlm"])


def load_expert_state_dict(root: Optional[str] = None) -> Dict[str, torch.Tensor]:
    root = root or DEFAULT_DIRS["expert"]
    return torch.load(os.path.join(root, "pytorch_model.bin"), map_location="cpu", weights_only=True)


def load_action_normalizer(root: Optional[str] = None) -> Dict[str, torch.Tensor]:
    """Returns {'scale': (20,), 'offset': (20,)} fp32; unnormalize(x) = (x - offset) / scale."""
    root = root or DEFAULT_DIRS["normalizer"]
    torch.serialization.add_safe_globals([torch.nn.modules.container.ParameterDict])
    pd = torch.load(os.path.join(root, NORMALIZER_FILE), map_location="cpu", weights_only=True)
    act = pd["action"]
    return {"scale": act["scale"].detach().float().clone(), "offset": act["offset"].detach().float().clone()}