changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
9.6 kB
# SPDX-License-Identifier: Apache-2.0
"""Gate sets and floors of the RDT2-FM device suite (megakernel workflow, VERIFY_PORT_r2 R2-F1 / F2 / F3).
TUNING set (may be used to choose a precision preset): the main golden (13 cases, bucket 416) and the ext golden
(5 cases: 10 * N(0, 1) states, 436 / 492-token prompts).
HOLD-OUT set (never used to choose anything; reported honestly, including failures and the ratio to the floor):
* the 4 round-2 verifier cases (``verify_r2_extra``: real 416 / 417 / 499 tokens, synthetic 484 tokens),
* 4 fresh cases of this workflow (``holdout``: two other real frames of the BimanualUR5e shard and a new synthetic
image; 420 / 463 / 467-token prompts; zero, N(0, 1) and 10 * N(0, 1) states; new noise seeds).
Every hold-out case has an official bf16 run.
Floor of a case (R2-F2): the larger of two bf16 pipelines' errors against the official fp32 run, per metric:
(a) the official bf16 run (``<set>/bf16/<case>.pt``),
(b) the bf16 torch reference expert (``ActionExpert(dtype=bf16)``) on the official bf16 K/V of that run.
Metrics: unnormalized group max-abs (position m / rotation-6D / gripper m: PRIMARY) and 1 - nsample PCC (SECONDARY).
``floors.json`` caches them (``compute_floors``); the device suite reads it.
"""
from __future__ import annotations
import json
import os
from typing import Dict, List, Optional, Tuple
import torch
from .config import DEFAULT_DIRS
TUNING_SETS = (("fp32", None), ("fp32", "ext")) # (run, sub-root under the golden dir)
HOLDOUT_SETS = ("verify_r2_extra", "holdout")
# 10 further official cases of an independent verification (real frames of other episodes, brightened / dimmed, a
# random image, constant and noise images, a Chinese / English instruction, 404..512-token prompts, states up to
# 10 * N(0, 1)); never used to choose anything. Not in floors.json: their floors are computed when needed
# (``floors_of``). The stress gate of the serving path judges them; the device suite does not.
VERIFIER_SETS = ("verify_mk_r1", "verify_mk_r2")
FLOORS_FILE = "floors.json"
def golden_root() -> str:
return DEFAULT_DIRS["golden"]
def group_maxabs(a: torch.Tensor, b: torch.Tensor) -> Tuple[float, float, float]:
"""Unnormalized action (24, 20): max-abs per group (position m, rotation-6D, gripper m) of a - b."""
d = (a.reshape(24, 20).double() - b.reshape(24, 20).double()).abs()
pos = torch.cat([d[:, 0:3], d[:, 10:13]], 1).max().item()
rot = torch.cat([d[:, 3:9], d[:, 13:19]], 1).max().item()
grip = torch.cat([d[:, 9:10], d[:, 19:20]], 1).max().item()
return pos, rot, grip
class HoldoutSet:
"""The hold-out cases: ``load(case)`` -> fp32 dump (with ``meta.prompt``), ``load_bf16(case)`` -> bf16 dump."""
def __init__(self, root: Optional[str] = None, sets: Tuple[str, ...] = HOLDOUT_SETS):
self.root = root or golden_root()
self.where: Dict[str, str] = {}
for sub in sets:
d = os.path.join(self.root, sub, "fp32")
if os.path.isdir(d):
for f in sorted(os.listdir(d)):
if f.endswith(".pt"):
self.where[f[:-3]] = sub
@property
def cases(self) -> List[str]:
return list(self.where)
def _load(self, case: str, run: str):
return torch.load(os.path.join(self.root, self.where[case], run, case + ".pt"), map_location="cpu",
weights_only=True)
def load(self, case: str):
return self._load(case, "fp32")
def has_bf16(self, case: str) -> bool:
return os.path.exists(os.path.join(self.root, self.where[case], "bf16", case + ".pt"))
def load_bf16(self, case: str):
return self._load(case, "bf16")
def _metrics(nsample, action, fp32) -> Dict[str, float]:
from .golden import pcc
pos, rot, grip = group_maxabs(action, fp32["out.action"])
return dict(one_minus_pcc=1.0 - pcc(nsample, fp32["exp.nsample"]), pos_m=pos, rot=rot, grip_m=grip)
def case_floor(fp32: Dict[str, torch.Tensor], bf16: Dict[str, torch.Tensor], expert_bf16, norm) -> Dict[str, object]:
"""Floor of one case: max over (official bf16 run, bf16 reference expert on the official bf16 K/V) per metric."""
from .expert import unnormalize_action
off = _metrics(bf16["exp.nsample"], bf16["out.action"], fp32)
n = bf16["cond.k"].shape[3]
ns = expert_bf16.sample(fp32["in.noise"], fp32["in.state"], bf16["cond.k"], bf16["cond.v"],
torch.ones(1, n, dtype=torch.bool)).float()
ref = _metrics(ns, unnormalize_action(ns, norm)[0], fp32)
floor = {k: max(off[k], ref[k]) for k in off}
return dict(official_bf16=off, bf16_ref_expert=ref, floor=floor)
# The bf16 reference-expert part of a floor is a bf16 CPU computation whose reduction order depends on the torch thread
# count (VERIFY_MK_r2 R2-F3: up to 43 % between runs). Floors are computed with this many threads; floors.json records it.
FLOOR_THREADS = 1
class _pinned_threads:
def __init__(self, n: int):
self.n = int(n)
def __enter__(self):
self.prev = torch.get_num_threads()
torch.set_num_threads(self.n)
return self
def __exit__(self, *exc):
torch.set_num_threads(self.prev)
return False
def compute_floors(out_path: Optional[str] = None, log=print, threads: int = FLOOR_THREADS
) -> Dict[str, Dict[str, object]]:
"""CPU: the floor of every case that has an official bf16 run (tuning main / ext and hold-out), with the torch
thread count pinned to ``threads`` (recorded per case as ``"threads"``)."""
with _pinned_threads(threads):
return _compute_floors(out_path, log, threads)
def _compute_floors(out_path: Optional[str], log, threads: int) -> Dict[str, Dict[str, object]]:
from .expert import ActionExpert
from .golden import GoldenSet
from .weights import load_action_normalizer, load_expert_state_dict
root = golden_root()
ex = ActionExpert(load_expert_state_dict(), dtype=torch.bfloat16)
norm = load_action_normalizer()
out: Dict[str, Dict[str, object]] = {}
pairs = []
for sub, kind in ((None, "tuning"), ("ext", "tuning")):
r = os.path.join(root, sub) if sub else root
if os.path.exists(os.path.join(r, "bf16", "manifest.json")):
g32, g16 = GoldenSet("fp32", root=r if sub else None), GoldenSet("bf16", root=r if sub else None)
for c in g16.cases:
d16 = g16.load(c, with_vlm=False)
if "cond.k" not in d16: # second-seed files share the first file's K/V
d16 = g16.load(c, with_vlm=True)
pairs.append((c, kind, g32.load(c), d16))
hs = HoldoutSet(root)
for c in hs.cases:
if hs.has_bf16(c):
pairs.append((c, "holdout", hs.load(c), hs.load_bf16(c)))
for c, kind, d32, d16 in pairs:
f = case_floor(d32, d16, ex, norm)
f["set"] = kind
f["threads"] = int(threads)
out[c] = f
fl = f["floor"]
log(f"[floor] {kind:8s} {c:24s} 1-PCC {fl['one_minus_pcc']:.3e} (official {f['official_bf16']['one_minus_pcc']:.3e}, "
f"bf16 ref {f['bf16_ref_expert']['one_minus_pcc']:.3e}) pos {fl['pos_m'] * 1e3:.2f} mm rot {fl['rot']:.4f} "
f"grip {fl['grip_m'] * 1e3:.2f} mm")
path = out_path or os.path.join(root, FLOORS_FILE)
with open(path, "w") as fh:
json.dump(out, fh, indent=1)
return out
def floors_of(cases: "HoldoutSet", names: List[str], expert_dir: Optional[str] = None,
normalizer_dir: Optional[str] = None) -> Dict[str, Dict[str, object]]:
"""CPU: the floors of ``names`` (cases of ``cases`` with an official bf16 run), computed like ``compute_floors``
without writing floors.json."""
from .expert import ActionExpert
from .weights import load_action_normalizer, load_expert_state_dict
names = [c for c in names if cases.has_bf16(c)]
if not names:
return {}
ex = ActionExpert(load_expert_state_dict(expert_dir), dtype=torch.bfloat16)
norm = load_action_normalizer(normalizer_dir)
with _pinned_threads(FLOOR_THREADS):
out = {c: case_floor(cases.load(c), cases.load_bf16(c), ex, norm) for c in names}
for f in out.values():
f["threads"] = FLOOR_THREADS
return out
def load_floors(path: Optional[str] = None) -> Dict[str, Dict[str, object]]:
path = path or os.path.join(golden_root(), FLOORS_FILE)
with open(path) as f:
return json.load(f)
# thresholds (VERIFY_PORT_r2 / PORT_LOG section 2): group max-abs primary, PCC secondary
FLOOR_X = 10.0 # 1 - PCC <= max(FLOOR_X * floor, FLOOR_MIN)
FLOOR_MIN = 1e-4
GRP_X = 3.0 # group max-abs <= max(GRP_X * floor, minimum)
GRP_MIN = dict(pos_m=0.005, rot=0.01, grip_m=0.0015)
def limits(floor: Dict[str, float]) -> Dict[str, float]:
return dict(one_minus_pcc=max(FLOOR_X * floor["one_minus_pcc"], FLOOR_MIN),
**{k: max(GRP_X * floor[k], GRP_MIN[k]) for k in ("pos_m", "rot", "grip_m")})
def judge(got: Dict[str, float], floor: Dict[str, float]) -> Dict[str, object]:
"""Floor-anchored A2 verdict of one case: groups first (primary), then PCC; ratios to the floor."""
lim = limits(floor)
ok_g = all(got[k] <= lim[k] for k in ("pos_m", "rot", "grip_m"))
ok_p = got["one_minus_pcc"] <= lim["one_minus_pcc"]
ratio = {k: got[k] / max(floor[k], 1e-12) for k in ("one_minus_pcc", "pos_m", "rot", "grip_m")}
return dict(limits=lim, groups_pass=ok_g, pcc_pass=ok_p, passed=ok_g and ok_p, ratio_to_floor=ratio)