File size: 3,177 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | # SPDX-License-Identifier: Apache-2.0
"""Weight loading straight from the pinned downloads (no official code, no transformers).
* VLM (RDT2-VQ, Qwen2.5-VL-7B): 4 bf16 safetensors shards + model.safetensors.index.json. Only the tensors a stage
needs are read (``safe_open`` per key); they stay bf16 in RAM and are upcast to fp32 on use (exact).
* Action expert (RDT2-FM): ``pytorch_model.bin`` (bf16 pickle) via ``torch.load(weights_only=True)``.
* Action normalizer: ``umi_normalizer_wo_downsample_indentity_rot.pt`` (pickled ``nn.ParameterDict``) via
``torch.load(weights_only=True)`` with ``ParameterDict`` allow-listed; only ``action.scale/offset`` are used.
"""
from __future__ import annotations
import json
import os
from typing import Dict, Iterable, Optional
import torch
from .config import DEFAULT_DIRS, NORMALIZER_FILE
class SafetensorsIndex:
"""Lazy reader of a sharded safetensors checkpoint (keys -> bf16 tensors, cached)."""
def __init__(self, root: str):
from safetensors import safe_open # local import: keep module import light
self._safe_open = safe_open
self.root = root
with open(os.path.join(root, "model.safetensors.index.json")) as f:
self.weight_map: Dict[str, str] = json.load(f)["weight_map"]
self._cache: Dict[str, torch.Tensor] = {}
def keys(self):
return self.weight_map.keys()
def load(self, names: Iterable[str]) -> Dict[str, torch.Tensor]:
by_file: Dict[str, list] = {}
for n in names:
if n not in self._cache:
by_file.setdefault(self.weight_map[n], []).append(n)
for fn, ns in by_file.items():
with self._safe_open(os.path.join(self.root, fn), framework="pt", device="cpu") as f:
for n in ns:
self._cache[n] = f.get_tensor(n)
return {n: self._cache[n] for n in names}
def load_prefix(self, prefix: str) -> Dict[str, torch.Tensor]:
return self.load([k for k in self.weight_map if k.startswith(prefix)])
def get(self, name: str) -> torch.Tensor:
return self.load([name])[name]
def drop(self, prefix: str = "") -> None:
for k in [k for k in self._cache if k.startswith(prefix)]:
del self._cache[k]
def load_vlm_index(root: Optional[str] = None) -> SafetensorsIndex:
return SafetensorsIndex(root or DEFAULT_DIRS["vlm"])
def load_expert_state_dict(root: Optional[str] = None) -> Dict[str, torch.Tensor]:
root = root or DEFAULT_DIRS["expert"]
return torch.load(os.path.join(root, "pytorch_model.bin"), map_location="cpu", weights_only=True)
def load_action_normalizer(root: Optional[str] = None) -> Dict[str, torch.Tensor]:
"""Returns {'scale': (20,), 'offset': (20,)} fp32; unnormalize(x) = (x - offset) / scale."""
root = root or DEFAULT_DIRS["normalizer"]
torch.serialization.add_safe_globals([torch.nn.modules.container.ParameterDict])
pd = torch.load(os.path.join(root, NORMALIZER_FILE), map_location="cpu", weights_only=True)
act = pd["action"]
return {"scale": act["scale"].detach().float().clone(), "offset": act["offset"].detach().float().clone()}
|