File size: 9,598 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
# SPDX-License-Identifier: Apache-2.0
"""Gate sets and floors of the RDT2-FM device suite (megakernel workflow, VERIFY_PORT_r2 R2-F1 / F2 / F3).

TUNING set (may be used to choose a precision preset): the main golden (13 cases, bucket 416) and the ext golden
(5 cases: 10 * N(0, 1) states, 436 / 492-token prompts).
HOLD-OUT set (never used to choose anything; reported honestly, including failures and the ratio to the floor):
  * the 4 round-2 verifier cases (``verify_r2_extra``: real 416 / 417 / 499 tokens, synthetic 484 tokens),
  * 4 fresh cases of this workflow (``holdout``: two other real frames of the BimanualUR5e shard and a new synthetic
    image; 420 / 463 / 467-token prompts; zero, N(0, 1) and 10 * N(0, 1) states; new noise seeds).
Every hold-out case has an official bf16 run.

Floor of a case (R2-F2): the larger of two bf16 pipelines' errors against the official fp32 run, per metric:
  (a) the official bf16 run (``<set>/bf16/<case>.pt``),
  (b) the bf16 torch reference expert (``ActionExpert(dtype=bf16)``) on the official bf16 K/V of that run.
Metrics: unnormalized group max-abs (position m / rotation-6D / gripper m: PRIMARY) and 1 - nsample PCC (SECONDARY).
``floors.json`` caches them (``compute_floors``); the device suite reads it.
"""
from __future__ import annotations

import json
import os
from typing import Dict, List, Optional, Tuple

import torch

from .config import DEFAULT_DIRS

TUNING_SETS = (("fp32", None), ("fp32", "ext"))  # (run, sub-root under the golden dir)
HOLDOUT_SETS = ("verify_r2_extra", "holdout")
# 10 further official cases of an independent verification (real frames of other episodes, brightened / dimmed, a
# random image, constant and noise images, a Chinese / English instruction, 404..512-token prompts, states up to
# 10 * N(0, 1)); never used to choose anything. Not in floors.json: their floors are computed when needed
# (``floors_of``). The stress gate of the serving path judges them; the device suite does not.
VERIFIER_SETS = ("verify_mk_r1", "verify_mk_r2")
FLOORS_FILE = "floors.json"


def golden_root() -> str:
    return DEFAULT_DIRS["golden"]


def group_maxabs(a: torch.Tensor, b: torch.Tensor) -> Tuple[float, float, float]:
    """Unnormalized action (24, 20): max-abs per group (position m, rotation-6D, gripper m) of a - b."""
    d = (a.reshape(24, 20).double() - b.reshape(24, 20).double()).abs()
    pos = torch.cat([d[:, 0:3], d[:, 10:13]], 1).max().item()
    rot = torch.cat([d[:, 3:9], d[:, 13:19]], 1).max().item()
    grip = torch.cat([d[:, 9:10], d[:, 19:20]], 1).max().item()
    return pos, rot, grip


class HoldoutSet:
    """The hold-out cases: ``load(case)`` -> fp32 dump (with ``meta.prompt``), ``load_bf16(case)`` -> bf16 dump."""

    def __init__(self, root: Optional[str] = None, sets: Tuple[str, ...] = HOLDOUT_SETS):
        self.root = root or golden_root()
        self.where: Dict[str, str] = {}
        for sub in sets:
            d = os.path.join(self.root, sub, "fp32")
            if os.path.isdir(d):
                for f in sorted(os.listdir(d)):
                    if f.endswith(".pt"):
                        self.where[f[:-3]] = sub

    @property
    def cases(self) -> List[str]:
        return list(self.where)

    def _load(self, case: str, run: str):
        return torch.load(os.path.join(self.root, self.where[case], run, case + ".pt"), map_location="cpu",
                          weights_only=True)

    def load(self, case: str):
        return self._load(case, "fp32")

    def has_bf16(self, case: str) -> bool:
        return os.path.exists(os.path.join(self.root, self.where[case], "bf16", case + ".pt"))

    def load_bf16(self, case: str):
        return self._load(case, "bf16")


def _metrics(nsample, action, fp32) -> Dict[str, float]:
    from .golden import pcc

    pos, rot, grip = group_maxabs(action, fp32["out.action"])
    return dict(one_minus_pcc=1.0 - pcc(nsample, fp32["exp.nsample"]), pos_m=pos, rot=rot, grip_m=grip)


def case_floor(fp32: Dict[str, torch.Tensor], bf16: Dict[str, torch.Tensor], expert_bf16, norm) -> Dict[str, object]:
    """Floor of one case: max over (official bf16 run, bf16 reference expert on the official bf16 K/V) per metric."""
    from .expert import unnormalize_action

    off = _metrics(bf16["exp.nsample"], bf16["out.action"], fp32)
    n = bf16["cond.k"].shape[3]
    ns = expert_bf16.sample(fp32["in.noise"], fp32["in.state"], bf16["cond.k"], bf16["cond.v"],
                            torch.ones(1, n, dtype=torch.bool)).float()
    ref = _metrics(ns, unnormalize_action(ns, norm)[0], fp32)
    floor = {k: max(off[k], ref[k]) for k in off}
    return dict(official_bf16=off, bf16_ref_expert=ref, floor=floor)


# The bf16 reference-expert part of a floor is a bf16 CPU computation whose reduction order depends on the torch thread
# count (VERIFY_MK_r2 R2-F3: up to 43 % between runs). Floors are computed with this many threads; floors.json records it.
FLOOR_THREADS = 1


class _pinned_threads:
    def __init__(self, n: int):
        self.n = int(n)

    def __enter__(self):
        self.prev = torch.get_num_threads()
        torch.set_num_threads(self.n)
        return self

    def __exit__(self, *exc):
        torch.set_num_threads(self.prev)
        return False


def compute_floors(out_path: Optional[str] = None, log=print, threads: int = FLOOR_THREADS
                   ) -> Dict[str, Dict[str, object]]:
    """CPU: the floor of every case that has an official bf16 run (tuning main / ext and hold-out), with the torch
    thread count pinned to ``threads`` (recorded per case as ``"threads"``)."""
    with _pinned_threads(threads):
        return _compute_floors(out_path, log, threads)


def _compute_floors(out_path: Optional[str], log, threads: int) -> Dict[str, Dict[str, object]]:
    from .expert import ActionExpert
    from .golden import GoldenSet
    from .weights import load_action_normalizer, load_expert_state_dict

    root = golden_root()
    ex = ActionExpert(load_expert_state_dict(), dtype=torch.bfloat16)
    norm = load_action_normalizer()
    out: Dict[str, Dict[str, object]] = {}
    pairs = []
    for sub, kind in ((None, "tuning"), ("ext", "tuning")):
        r = os.path.join(root, sub) if sub else root
        if os.path.exists(os.path.join(r, "bf16", "manifest.json")):
            g32, g16 = GoldenSet("fp32", root=r if sub else None), GoldenSet("bf16", root=r if sub else None)
            for c in g16.cases:
                d16 = g16.load(c, with_vlm=False)
                if "cond.k" not in d16:  # second-seed files share the first file's K/V
                    d16 = g16.load(c, with_vlm=True)
                pairs.append((c, kind, g32.load(c), d16))
    hs = HoldoutSet(root)
    for c in hs.cases:
        if hs.has_bf16(c):
            pairs.append((c, "holdout", hs.load(c), hs.load_bf16(c)))
    for c, kind, d32, d16 in pairs:
        f = case_floor(d32, d16, ex, norm)
        f["set"] = kind
        f["threads"] = int(threads)
        out[c] = f
        fl = f["floor"]
        log(f"[floor] {kind:8s} {c:24s} 1-PCC {fl['one_minus_pcc']:.3e} (official {f['official_bf16']['one_minus_pcc']:.3e}, "
            f"bf16 ref {f['bf16_ref_expert']['one_minus_pcc']:.3e})  pos {fl['pos_m'] * 1e3:.2f} mm rot {fl['rot']:.4f} "
            f"grip {fl['grip_m'] * 1e3:.2f} mm")
    path = out_path or os.path.join(root, FLOORS_FILE)
    with open(path, "w") as fh:
        json.dump(out, fh, indent=1)
    return out


def floors_of(cases: "HoldoutSet", names: List[str], expert_dir: Optional[str] = None,
              normalizer_dir: Optional[str] = None) -> Dict[str, Dict[str, object]]:
    """CPU: the floors of ``names`` (cases of ``cases`` with an official bf16 run), computed like ``compute_floors``
    without writing floors.json."""
    from .expert import ActionExpert
    from .weights import load_action_normalizer, load_expert_state_dict

    names = [c for c in names if cases.has_bf16(c)]
    if not names:
        return {}
    ex = ActionExpert(load_expert_state_dict(expert_dir), dtype=torch.bfloat16)
    norm = load_action_normalizer(normalizer_dir)
    with _pinned_threads(FLOOR_THREADS):
        out = {c: case_floor(cases.load(c), cases.load_bf16(c), ex, norm) for c in names}
    for f in out.values():
        f["threads"] = FLOOR_THREADS
    return out


def load_floors(path: Optional[str] = None) -> Dict[str, Dict[str, object]]:
    path = path or os.path.join(golden_root(), FLOORS_FILE)
    with open(path) as f:
        return json.load(f)


# thresholds (VERIFY_PORT_r2 / PORT_LOG section 2): group max-abs primary, PCC secondary
FLOOR_X = 10.0       # 1 - PCC <= max(FLOOR_X * floor, FLOOR_MIN)
FLOOR_MIN = 1e-4
GRP_X = 3.0          # group max-abs <= max(GRP_X * floor, minimum)
GRP_MIN = dict(pos_m=0.005, rot=0.01, grip_m=0.0015)


def limits(floor: Dict[str, float]) -> Dict[str, float]:
    return dict(one_minus_pcc=max(FLOOR_X * floor["one_minus_pcc"], FLOOR_MIN),
                **{k: max(GRP_X * floor[k], GRP_MIN[k]) for k in ("pos_m", "rot", "grip_m")})


def judge(got: Dict[str, float], floor: Dict[str, float]) -> Dict[str, object]:
    """Floor-anchored A2 verdict of one case: groups first (primary), then PCC; ratios to the floor."""
    lim = limits(floor)
    ok_g = all(got[k] <= lim[k] for k in ("pos_m", "rot", "grip_m"))
    ok_p = got["one_minus_pcc"] <= lim["one_minus_pcc"]
    ratio = {k: got[k] / max(floor[k], 1e-12) for k in ("one_minus_pcc", "pos_m", "rot", "grip_m")}
    return dict(limits=lim, groups_pass=ok_g, pcc_pass=ok_p, passed=ok_g and ok_p, ratio_to_floor=ratio)