# SPDX-License-Identifier: Apache-2.0 """Gate sets and floors of the RDT2-FM device suite (megakernel workflow, VERIFY_PORT_r2 R2-F1 / F2 / F3). TUNING set (may be used to choose a precision preset): the main golden (13 cases, bucket 416) and the ext golden (5 cases: 10 * N(0, 1) states, 436 / 492-token prompts). HOLD-OUT set (never used to choose anything; reported honestly, including failures and the ratio to the floor): * the 4 round-2 verifier cases (``verify_r2_extra``: real 416 / 417 / 499 tokens, synthetic 484 tokens), * 4 fresh cases of this workflow (``holdout``: two other real frames of the BimanualUR5e shard and a new synthetic image; 420 / 463 / 467-token prompts; zero, N(0, 1) and 10 * N(0, 1) states; new noise seeds). Every hold-out case has an official bf16 run. Floor of a case (R2-F2): the larger of two bf16 pipelines' errors against the official fp32 run, per metric: (a) the official bf16 run (``/bf16/.pt``), (b) the bf16 torch reference expert (``ActionExpert(dtype=bf16)``) on the official bf16 K/V of that run. Metrics: unnormalized group max-abs (position m / rotation-6D / gripper m: PRIMARY) and 1 - nsample PCC (SECONDARY). ``floors.json`` caches them (``compute_floors``); the device suite reads it. """ from __future__ import annotations import json import os from typing import Dict, List, Optional, Tuple import torch from .config import DEFAULT_DIRS TUNING_SETS = (("fp32", None), ("fp32", "ext")) # (run, sub-root under the golden dir) HOLDOUT_SETS = ("verify_r2_extra", "holdout") # 10 further official cases of an independent verification (real frames of other episodes, brightened / dimmed, a # random image, constant and noise images, a Chinese / English instruction, 404..512-token prompts, states up to # 10 * N(0, 1)); never used to choose anything. Not in floors.json: their floors are computed when needed # (``floors_of``). The stress gate of the serving path judges them; the device suite does not. VERIFIER_SETS = ("verify_mk_r1", "verify_mk_r2") FLOORS_FILE = "floors.json" def golden_root() -> str: return DEFAULT_DIRS["golden"] def group_maxabs(a: torch.Tensor, b: torch.Tensor) -> Tuple[float, float, float]: """Unnormalized action (24, 20): max-abs per group (position m, rotation-6D, gripper m) of a - b.""" d = (a.reshape(24, 20).double() - b.reshape(24, 20).double()).abs() pos = torch.cat([d[:, 0:3], d[:, 10:13]], 1).max().item() rot = torch.cat([d[:, 3:9], d[:, 13:19]], 1).max().item() grip = torch.cat([d[:, 9:10], d[:, 19:20]], 1).max().item() return pos, rot, grip class HoldoutSet: """The hold-out cases: ``load(case)`` -> fp32 dump (with ``meta.prompt``), ``load_bf16(case)`` -> bf16 dump.""" def __init__(self, root: Optional[str] = None, sets: Tuple[str, ...] = HOLDOUT_SETS): self.root = root or golden_root() self.where: Dict[str, str] = {} for sub in sets: d = os.path.join(self.root, sub, "fp32") if os.path.isdir(d): for f in sorted(os.listdir(d)): if f.endswith(".pt"): self.where[f[:-3]] = sub @property def cases(self) -> List[str]: return list(self.where) def _load(self, case: str, run: str): return torch.load(os.path.join(self.root, self.where[case], run, case + ".pt"), map_location="cpu", weights_only=True) def load(self, case: str): return self._load(case, "fp32") def has_bf16(self, case: str) -> bool: return os.path.exists(os.path.join(self.root, self.where[case], "bf16", case + ".pt")) def load_bf16(self, case: str): return self._load(case, "bf16") def _metrics(nsample, action, fp32) -> Dict[str, float]: from .golden import pcc pos, rot, grip = group_maxabs(action, fp32["out.action"]) return dict(one_minus_pcc=1.0 - pcc(nsample, fp32["exp.nsample"]), pos_m=pos, rot=rot, grip_m=grip) def case_floor(fp32: Dict[str, torch.Tensor], bf16: Dict[str, torch.Tensor], expert_bf16, norm) -> Dict[str, object]: """Floor of one case: max over (official bf16 run, bf16 reference expert on the official bf16 K/V) per metric.""" from .expert import unnormalize_action off = _metrics(bf16["exp.nsample"], bf16["out.action"], fp32) n = bf16["cond.k"].shape[3] ns = expert_bf16.sample(fp32["in.noise"], fp32["in.state"], bf16["cond.k"], bf16["cond.v"], torch.ones(1, n, dtype=torch.bool)).float() ref = _metrics(ns, unnormalize_action(ns, norm)[0], fp32) floor = {k: max(off[k], ref[k]) for k in off} return dict(official_bf16=off, bf16_ref_expert=ref, floor=floor) # The bf16 reference-expert part of a floor is a bf16 CPU computation whose reduction order depends on the torch thread # count (VERIFY_MK_r2 R2-F3: up to 43 % between runs). Floors are computed with this many threads; floors.json records it. FLOOR_THREADS = 1 class _pinned_threads: def __init__(self, n: int): self.n = int(n) def __enter__(self): self.prev = torch.get_num_threads() torch.set_num_threads(self.n) return self def __exit__(self, *exc): torch.set_num_threads(self.prev) return False def compute_floors(out_path: Optional[str] = None, log=print, threads: int = FLOOR_THREADS ) -> Dict[str, Dict[str, object]]: """CPU: the floor of every case that has an official bf16 run (tuning main / ext and hold-out), with the torch thread count pinned to ``threads`` (recorded per case as ``"threads"``).""" with _pinned_threads(threads): return _compute_floors(out_path, log, threads) def _compute_floors(out_path: Optional[str], log, threads: int) -> Dict[str, Dict[str, object]]: from .expert import ActionExpert from .golden import GoldenSet from .weights import load_action_normalizer, load_expert_state_dict root = golden_root() ex = ActionExpert(load_expert_state_dict(), dtype=torch.bfloat16) norm = load_action_normalizer() out: Dict[str, Dict[str, object]] = {} pairs = [] for sub, kind in ((None, "tuning"), ("ext", "tuning")): r = os.path.join(root, sub) if sub else root if os.path.exists(os.path.join(r, "bf16", "manifest.json")): g32, g16 = GoldenSet("fp32", root=r if sub else None), GoldenSet("bf16", root=r if sub else None) for c in g16.cases: d16 = g16.load(c, with_vlm=False) if "cond.k" not in d16: # second-seed files share the first file's K/V d16 = g16.load(c, with_vlm=True) pairs.append((c, kind, g32.load(c), d16)) hs = HoldoutSet(root) for c in hs.cases: if hs.has_bf16(c): pairs.append((c, "holdout", hs.load(c), hs.load_bf16(c))) for c, kind, d32, d16 in pairs: f = case_floor(d32, d16, ex, norm) f["set"] = kind f["threads"] = int(threads) out[c] = f fl = f["floor"] log(f"[floor] {kind:8s} {c:24s} 1-PCC {fl['one_minus_pcc']:.3e} (official {f['official_bf16']['one_minus_pcc']:.3e}, " f"bf16 ref {f['bf16_ref_expert']['one_minus_pcc']:.3e}) pos {fl['pos_m'] * 1e3:.2f} mm rot {fl['rot']:.4f} " f"grip {fl['grip_m'] * 1e3:.2f} mm") path = out_path or os.path.join(root, FLOORS_FILE) with open(path, "w") as fh: json.dump(out, fh, indent=1) return out def floors_of(cases: "HoldoutSet", names: List[str], expert_dir: Optional[str] = None, normalizer_dir: Optional[str] = None) -> Dict[str, Dict[str, object]]: """CPU: the floors of ``names`` (cases of ``cases`` with an official bf16 run), computed like ``compute_floors`` without writing floors.json.""" from .expert import ActionExpert from .weights import load_action_normalizer, load_expert_state_dict names = [c for c in names if cases.has_bf16(c)] if not names: return {} ex = ActionExpert(load_expert_state_dict(expert_dir), dtype=torch.bfloat16) norm = load_action_normalizer(normalizer_dir) with _pinned_threads(FLOOR_THREADS): out = {c: case_floor(cases.load(c), cases.load_bf16(c), ex, norm) for c in names} for f in out.values(): f["threads"] = FLOOR_THREADS return out def load_floors(path: Optional[str] = None) -> Dict[str, Dict[str, object]]: path = path or os.path.join(golden_root(), FLOORS_FILE) with open(path) as f: return json.load(f) # thresholds (VERIFY_PORT_r2 / PORT_LOG section 2): group max-abs primary, PCC secondary FLOOR_X = 10.0 # 1 - PCC <= max(FLOOR_X * floor, FLOOR_MIN) FLOOR_MIN = 1e-4 GRP_X = 3.0 # group max-abs <= max(GRP_X * floor, minimum) GRP_MIN = dict(pos_m=0.005, rot=0.01, grip_m=0.0015) def limits(floor: Dict[str, float]) -> Dict[str, float]: return dict(one_minus_pcc=max(FLOOR_X * floor["one_minus_pcc"], FLOOR_MIN), **{k: max(GRP_X * floor[k], GRP_MIN[k]) for k in ("pos_m", "rot", "grip_m")}) def judge(got: Dict[str, float], floor: Dict[str, float]) -> Dict[str, object]: """Floor-anchored A2 verdict of one case: groups first (primary), then PCC; ratios to the floor.""" lim = limits(floor) ok_g = all(got[k] <= lim[k] for k in ("pos_m", "rot", "grip_m")) ok_p = got["one_minus_pcc"] <= lim["one_minus_pcc"] ratio = {k: got[k] / max(floor[k], 1e-12) for k in ("one_minus_pcc", "pos_m", "rot", "grip_m")} return dict(limits=lim, groups_pass=ok_g, pcc_pass=ok_p, passed=ok_g and ok_p, ratio_to_floor=ratio)