File size: 9,598 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | # SPDX-License-Identifier: Apache-2.0
"""Gate sets and floors of the RDT2-FM device suite (megakernel workflow, VERIFY_PORT_r2 R2-F1 / F2 / F3).
TUNING set (may be used to choose a precision preset): the main golden (13 cases, bucket 416) and the ext golden
(5 cases: 10 * N(0, 1) states, 436 / 492-token prompts).
HOLD-OUT set (never used to choose anything; reported honestly, including failures and the ratio to the floor):
* the 4 round-2 verifier cases (``verify_r2_extra``: real 416 / 417 / 499 tokens, synthetic 484 tokens),
* 4 fresh cases of this workflow (``holdout``: two other real frames of the BimanualUR5e shard and a new synthetic
image; 420 / 463 / 467-token prompts; zero, N(0, 1) and 10 * N(0, 1) states; new noise seeds).
Every hold-out case has an official bf16 run.
Floor of a case (R2-F2): the larger of two bf16 pipelines' errors against the official fp32 run, per metric:
(a) the official bf16 run (``<set>/bf16/<case>.pt``),
(b) the bf16 torch reference expert (``ActionExpert(dtype=bf16)``) on the official bf16 K/V of that run.
Metrics: unnormalized group max-abs (position m / rotation-6D / gripper m: PRIMARY) and 1 - nsample PCC (SECONDARY).
``floors.json`` caches them (``compute_floors``); the device suite reads it.
"""
from __future__ import annotations
import json
import os
from typing import Dict, List, Optional, Tuple
import torch
from .config import DEFAULT_DIRS
TUNING_SETS = (("fp32", None), ("fp32", "ext")) # (run, sub-root under the golden dir)
HOLDOUT_SETS = ("verify_r2_extra", "holdout")
# 10 further official cases of an independent verification (real frames of other episodes, brightened / dimmed, a
# random image, constant and noise images, a Chinese / English instruction, 404..512-token prompts, states up to
# 10 * N(0, 1)); never used to choose anything. Not in floors.json: their floors are computed when needed
# (``floors_of``). The stress gate of the serving path judges them; the device suite does not.
VERIFIER_SETS = ("verify_mk_r1", "verify_mk_r2")
FLOORS_FILE = "floors.json"
def golden_root() -> str:
return DEFAULT_DIRS["golden"]
def group_maxabs(a: torch.Tensor, b: torch.Tensor) -> Tuple[float, float, float]:
"""Unnormalized action (24, 20): max-abs per group (position m, rotation-6D, gripper m) of a - b."""
d = (a.reshape(24, 20).double() - b.reshape(24, 20).double()).abs()
pos = torch.cat([d[:, 0:3], d[:, 10:13]], 1).max().item()
rot = torch.cat([d[:, 3:9], d[:, 13:19]], 1).max().item()
grip = torch.cat([d[:, 9:10], d[:, 19:20]], 1).max().item()
return pos, rot, grip
class HoldoutSet:
"""The hold-out cases: ``load(case)`` -> fp32 dump (with ``meta.prompt``), ``load_bf16(case)`` -> bf16 dump."""
def __init__(self, root: Optional[str] = None, sets: Tuple[str, ...] = HOLDOUT_SETS):
self.root = root or golden_root()
self.where: Dict[str, str] = {}
for sub in sets:
d = os.path.join(self.root, sub, "fp32")
if os.path.isdir(d):
for f in sorted(os.listdir(d)):
if f.endswith(".pt"):
self.where[f[:-3]] = sub
@property
def cases(self) -> List[str]:
return list(self.where)
def _load(self, case: str, run: str):
return torch.load(os.path.join(self.root, self.where[case], run, case + ".pt"), map_location="cpu",
weights_only=True)
def load(self, case: str):
return self._load(case, "fp32")
def has_bf16(self, case: str) -> bool:
return os.path.exists(os.path.join(self.root, self.where[case], "bf16", case + ".pt"))
def load_bf16(self, case: str):
return self._load(case, "bf16")
def _metrics(nsample, action, fp32) -> Dict[str, float]:
from .golden import pcc
pos, rot, grip = group_maxabs(action, fp32["out.action"])
return dict(one_minus_pcc=1.0 - pcc(nsample, fp32["exp.nsample"]), pos_m=pos, rot=rot, grip_m=grip)
def case_floor(fp32: Dict[str, torch.Tensor], bf16: Dict[str, torch.Tensor], expert_bf16, norm) -> Dict[str, object]:
"""Floor of one case: max over (official bf16 run, bf16 reference expert on the official bf16 K/V) per metric."""
from .expert import unnormalize_action
off = _metrics(bf16["exp.nsample"], bf16["out.action"], fp32)
n = bf16["cond.k"].shape[3]
ns = expert_bf16.sample(fp32["in.noise"], fp32["in.state"], bf16["cond.k"], bf16["cond.v"],
torch.ones(1, n, dtype=torch.bool)).float()
ref = _metrics(ns, unnormalize_action(ns, norm)[0], fp32)
floor = {k: max(off[k], ref[k]) for k in off}
return dict(official_bf16=off, bf16_ref_expert=ref, floor=floor)
# The bf16 reference-expert part of a floor is a bf16 CPU computation whose reduction order depends on the torch thread
# count (VERIFY_MK_r2 R2-F3: up to 43 % between runs). Floors are computed with this many threads; floors.json records it.
FLOOR_THREADS = 1
class _pinned_threads:
def __init__(self, n: int):
self.n = int(n)
def __enter__(self):
self.prev = torch.get_num_threads()
torch.set_num_threads(self.n)
return self
def __exit__(self, *exc):
torch.set_num_threads(self.prev)
return False
def compute_floors(out_path: Optional[str] = None, log=print, threads: int = FLOOR_THREADS
) -> Dict[str, Dict[str, object]]:
"""CPU: the floor of every case that has an official bf16 run (tuning main / ext and hold-out), with the torch
thread count pinned to ``threads`` (recorded per case as ``"threads"``)."""
with _pinned_threads(threads):
return _compute_floors(out_path, log, threads)
def _compute_floors(out_path: Optional[str], log, threads: int) -> Dict[str, Dict[str, object]]:
from .expert import ActionExpert
from .golden import GoldenSet
from .weights import load_action_normalizer, load_expert_state_dict
root = golden_root()
ex = ActionExpert(load_expert_state_dict(), dtype=torch.bfloat16)
norm = load_action_normalizer()
out: Dict[str, Dict[str, object]] = {}
pairs = []
for sub, kind in ((None, "tuning"), ("ext", "tuning")):
r = os.path.join(root, sub) if sub else root
if os.path.exists(os.path.join(r, "bf16", "manifest.json")):
g32, g16 = GoldenSet("fp32", root=r if sub else None), GoldenSet("bf16", root=r if sub else None)
for c in g16.cases:
d16 = g16.load(c, with_vlm=False)
if "cond.k" not in d16: # second-seed files share the first file's K/V
d16 = g16.load(c, with_vlm=True)
pairs.append((c, kind, g32.load(c), d16))
hs = HoldoutSet(root)
for c in hs.cases:
if hs.has_bf16(c):
pairs.append((c, "holdout", hs.load(c), hs.load_bf16(c)))
for c, kind, d32, d16 in pairs:
f = case_floor(d32, d16, ex, norm)
f["set"] = kind
f["threads"] = int(threads)
out[c] = f
fl = f["floor"]
log(f"[floor] {kind:8s} {c:24s} 1-PCC {fl['one_minus_pcc']:.3e} (official {f['official_bf16']['one_minus_pcc']:.3e}, "
f"bf16 ref {f['bf16_ref_expert']['one_minus_pcc']:.3e}) pos {fl['pos_m'] * 1e3:.2f} mm rot {fl['rot']:.4f} "
f"grip {fl['grip_m'] * 1e3:.2f} mm")
path = out_path or os.path.join(root, FLOORS_FILE)
with open(path, "w") as fh:
json.dump(out, fh, indent=1)
return out
def floors_of(cases: "HoldoutSet", names: List[str], expert_dir: Optional[str] = None,
normalizer_dir: Optional[str] = None) -> Dict[str, Dict[str, object]]:
"""CPU: the floors of ``names`` (cases of ``cases`` with an official bf16 run), computed like ``compute_floors``
without writing floors.json."""
from .expert import ActionExpert
from .weights import load_action_normalizer, load_expert_state_dict
names = [c for c in names if cases.has_bf16(c)]
if not names:
return {}
ex = ActionExpert(load_expert_state_dict(expert_dir), dtype=torch.bfloat16)
norm = load_action_normalizer(normalizer_dir)
with _pinned_threads(FLOOR_THREADS):
out = {c: case_floor(cases.load(c), cases.load_bf16(c), ex, norm) for c in names}
for f in out.values():
f["threads"] = FLOOR_THREADS
return out
def load_floors(path: Optional[str] = None) -> Dict[str, Dict[str, object]]:
path = path or os.path.join(golden_root(), FLOORS_FILE)
with open(path) as f:
return json.load(f)
# thresholds (VERIFY_PORT_r2 / PORT_LOG section 2): group max-abs primary, PCC secondary
FLOOR_X = 10.0 # 1 - PCC <= max(FLOOR_X * floor, FLOOR_MIN)
FLOOR_MIN = 1e-4
GRP_X = 3.0 # group max-abs <= max(GRP_X * floor, minimum)
GRP_MIN = dict(pos_m=0.005, rot=0.01, grip_m=0.0015)
def limits(floor: Dict[str, float]) -> Dict[str, float]:
return dict(one_minus_pcc=max(FLOOR_X * floor["one_minus_pcc"], FLOOR_MIN),
**{k: max(GRP_X * floor[k], GRP_MIN[k]) for k in ("pos_m", "rot", "grip_m")})
def judge(got: Dict[str, float], floor: Dict[str, float]) -> Dict[str, object]:
"""Floor-anchored A2 verdict of one case: groups first (primary), then PCC; ratios to the floor."""
lim = limits(floor)
ok_g = all(got[k] <= lim[k] for k in ("pos_m", "rot", "grip_m"))
ok_p = got["one_minus_pcc"] <= lim["one_minus_pcc"]
ratio = {k: got[k] / max(floor[k], 1e-12) for k in ("one_minus_pcc", "pos_m", "rot", "grip_m")}
return dict(limits=lim, groups_pass=ok_g, pcc_pass=ok_p, passed=ok_g and ok_p, ratio_to_floor=ratio)
|