Download code/tt_rdt2/reference/gatesets.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 9.6 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/gatesets.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/gatesets.py
-
curl -L -o gatesets.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/gatesets.py
9.6 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Gate sets and floors of the RDT2-FM device suite (megakernel workflow, VERIFY_PORT_r2 R2-F1 / F2 / F3). | |
| TUNING set (may be used to choose a precision preset): the main golden (13 cases, bucket 416) and the ext golden | |
| (5 cases: 10 * N(0, 1) states, 436 / 492-token prompts). | |
| HOLD-OUT set (never used to choose anything; reported honestly, including failures and the ratio to the floor): | |
| * the 4 round-2 verifier cases (``verify_r2_extra``: real 416 / 417 / 499 tokens, synthetic 484 tokens), | |
| * 4 fresh cases of this workflow (``holdout``: two other real frames of the BimanualUR5e shard and a new synthetic | |
| image; 420 / 463 / 467-token prompts; zero, N(0, 1) and 10 * N(0, 1) states; new noise seeds). | |
| Every hold-out case has an official bf16 run. | |
| Floor of a case (R2-F2): the larger of two bf16 pipelines' errors against the official fp32 run, per metric: | |
| (a) the official bf16 run (``<set>/bf16/<case>.pt``), | |
| (b) the bf16 torch reference expert (``ActionExpert(dtype=bf16)``) on the official bf16 K/V of that run. | |
| Metrics: unnormalized group max-abs (position m / rotation-6D / gripper m: PRIMARY) and 1 - nsample PCC (SECONDARY). | |
| ``floors.json`` caches them (``compute_floors``); the device suite reads it. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| from typing import Dict, List, Optional, Tuple | |
| import torch | |
| from .config import DEFAULT_DIRS | |
| TUNING_SETS = (("fp32", None), ("fp32", "ext")) # (run, sub-root under the golden dir) | |
| HOLDOUT_SETS = ("verify_r2_extra", "holdout") | |
| # 10 further official cases of an independent verification (real frames of other episodes, brightened / dimmed, a | |
| # random image, constant and noise images, a Chinese / English instruction, 404..512-token prompts, states up to | |
| # 10 * N(0, 1)); never used to choose anything. Not in floors.json: their floors are computed when needed | |
| # (``floors_of``). The stress gate of the serving path judges them; the device suite does not. | |
| VERIFIER_SETS = ("verify_mk_r1", "verify_mk_r2") | |
| FLOORS_FILE = "floors.json" | |
| def golden_root() -> str: | |
| return DEFAULT_DIRS["golden"] | |
| def group_maxabs(a: torch.Tensor, b: torch.Tensor) -> Tuple[float, float, float]: | |
| """Unnormalized action (24, 20): max-abs per group (position m, rotation-6D, gripper m) of a - b.""" | |
| d = (a.reshape(24, 20).double() - b.reshape(24, 20).double()).abs() | |
| pos = torch.cat([d[:, 0:3], d[:, 10:13]], 1).max().item() | |
| rot = torch.cat([d[:, 3:9], d[:, 13:19]], 1).max().item() | |
| grip = torch.cat([d[:, 9:10], d[:, 19:20]], 1).max().item() | |
| return pos, rot, grip | |
| class HoldoutSet: | |
| """The hold-out cases: ``load(case)`` -> fp32 dump (with ``meta.prompt``), ``load_bf16(case)`` -> bf16 dump.""" | |
| def __init__(self, root: Optional[str] = None, sets: Tuple[str, ...] = HOLDOUT_SETS): | |
| self.root = root or golden_root() | |
| self.where: Dict[str, str] = {} | |
| for sub in sets: | |
| d = os.path.join(self.root, sub, "fp32") | |
| if os.path.isdir(d): | |
| for f in sorted(os.listdir(d)): | |
| if f.endswith(".pt"): | |
| self.where[f[:-3]] = sub | |
| def cases(self) -> List[str]: | |
| return list(self.where) | |
| def _load(self, case: str, run: str): | |
| return torch.load(os.path.join(self.root, self.where[case], run, case + ".pt"), map_location="cpu", | |
| weights_only=True) | |
| def load(self, case: str): | |
| return self._load(case, "fp32") | |
| def has_bf16(self, case: str) -> bool: | |
| return os.path.exists(os.path.join(self.root, self.where[case], "bf16", case + ".pt")) | |
| def load_bf16(self, case: str): | |
| return self._load(case, "bf16") | |
| def _metrics(nsample, action, fp32) -> Dict[str, float]: | |
| from .golden import pcc | |
| pos, rot, grip = group_maxabs(action, fp32["out.action"]) | |
| return dict(one_minus_pcc=1.0 - pcc(nsample, fp32["exp.nsample"]), pos_m=pos, rot=rot, grip_m=grip) | |
| def case_floor(fp32: Dict[str, torch.Tensor], bf16: Dict[str, torch.Tensor], expert_bf16, norm) -> Dict[str, object]: | |
| """Floor of one case: max over (official bf16 run, bf16 reference expert on the official bf16 K/V) per metric.""" | |
| from .expert import unnormalize_action | |
| off = _metrics(bf16["exp.nsample"], bf16["out.action"], fp32) | |
| n = bf16["cond.k"].shape[3] | |
| ns = expert_bf16.sample(fp32["in.noise"], fp32["in.state"], bf16["cond.k"], bf16["cond.v"], | |
| torch.ones(1, n, dtype=torch.bool)).float() | |
| ref = _metrics(ns, unnormalize_action(ns, norm)[0], fp32) | |
| floor = {k: max(off[k], ref[k]) for k in off} | |
| return dict(official_bf16=off, bf16_ref_expert=ref, floor=floor) | |
| # The bf16 reference-expert part of a floor is a bf16 CPU computation whose reduction order depends on the torch thread | |
| # count (VERIFY_MK_r2 R2-F3: up to 43 % between runs). Floors are computed with this many threads; floors.json records it. | |
| FLOOR_THREADS = 1 | |
| class _pinned_threads: | |
| def __init__(self, n: int): | |
| self.n = int(n) | |
| def __enter__(self): | |
| self.prev = torch.get_num_threads() | |
| torch.set_num_threads(self.n) | |
| return self | |
| def __exit__(self, *exc): | |
| torch.set_num_threads(self.prev) | |
| return False | |
| def compute_floors(out_path: Optional[str] = None, log=print, threads: int = FLOOR_THREADS | |
| ) -> Dict[str, Dict[str, object]]: | |
| """CPU: the floor of every case that has an official bf16 run (tuning main / ext and hold-out), with the torch | |
| thread count pinned to ``threads`` (recorded per case as ``"threads"``).""" | |
| with _pinned_threads(threads): | |
| return _compute_floors(out_path, log, threads) | |
| def _compute_floors(out_path: Optional[str], log, threads: int) -> Dict[str, Dict[str, object]]: | |
| from .expert import ActionExpert | |
| from .golden import GoldenSet | |
| from .weights import load_action_normalizer, load_expert_state_dict | |
| root = golden_root() | |
| ex = ActionExpert(load_expert_state_dict(), dtype=torch.bfloat16) | |
| norm = load_action_normalizer() | |
| out: Dict[str, Dict[str, object]] = {} | |
| pairs = [] | |
| for sub, kind in ((None, "tuning"), ("ext", "tuning")): | |
| r = os.path.join(root, sub) if sub else root | |
| if os.path.exists(os.path.join(r, "bf16", "manifest.json")): | |
| g32, g16 = GoldenSet("fp32", root=r if sub else None), GoldenSet("bf16", root=r if sub else None) | |
| for c in g16.cases: | |
| d16 = g16.load(c, with_vlm=False) | |
| if "cond.k" not in d16: # second-seed files share the first file's K/V | |
| d16 = g16.load(c, with_vlm=True) | |
| pairs.append((c, kind, g32.load(c), d16)) | |
| hs = HoldoutSet(root) | |
| for c in hs.cases: | |
| if hs.has_bf16(c): | |
| pairs.append((c, "holdout", hs.load(c), hs.load_bf16(c))) | |
| for c, kind, d32, d16 in pairs: | |
| f = case_floor(d32, d16, ex, norm) | |
| f["set"] = kind | |
| f["threads"] = int(threads) | |
| out[c] = f | |
| fl = f["floor"] | |
| log(f"[floor] {kind:8s} {c:24s} 1-PCC {fl['one_minus_pcc']:.3e} (official {f['official_bf16']['one_minus_pcc']:.3e}, " | |
| f"bf16 ref {f['bf16_ref_expert']['one_minus_pcc']:.3e}) pos {fl['pos_m'] * 1e3:.2f} mm rot {fl['rot']:.4f} " | |
| f"grip {fl['grip_m'] * 1e3:.2f} mm") | |
| path = out_path or os.path.join(root, FLOORS_FILE) | |
| with open(path, "w") as fh: | |
| json.dump(out, fh, indent=1) | |
| return out | |
| def floors_of(cases: "HoldoutSet", names: List[str], expert_dir: Optional[str] = None, | |
| normalizer_dir: Optional[str] = None) -> Dict[str, Dict[str, object]]: | |
| """CPU: the floors of ``names`` (cases of ``cases`` with an official bf16 run), computed like ``compute_floors`` | |
| without writing floors.json.""" | |
| from .expert import ActionExpert | |
| from .weights import load_action_normalizer, load_expert_state_dict | |
| names = [c for c in names if cases.has_bf16(c)] | |
| if not names: | |
| return {} | |
| ex = ActionExpert(load_expert_state_dict(expert_dir), dtype=torch.bfloat16) | |
| norm = load_action_normalizer(normalizer_dir) | |
| with _pinned_threads(FLOOR_THREADS): | |
| out = {c: case_floor(cases.load(c), cases.load_bf16(c), ex, norm) for c in names} | |
| for f in out.values(): | |
| f["threads"] = FLOOR_THREADS | |
| return out | |
| def load_floors(path: Optional[str] = None) -> Dict[str, Dict[str, object]]: | |
| path = path or os.path.join(golden_root(), FLOORS_FILE) | |
| with open(path) as f: | |
| return json.load(f) | |
| # thresholds (VERIFY_PORT_r2 / PORT_LOG section 2): group max-abs primary, PCC secondary | |
| FLOOR_X = 10.0 # 1 - PCC <= max(FLOOR_X * floor, FLOOR_MIN) | |
| FLOOR_MIN = 1e-4 | |
| GRP_X = 3.0 # group max-abs <= max(GRP_X * floor, minimum) | |
| GRP_MIN = dict(pos_m=0.005, rot=0.01, grip_m=0.0015) | |
| def limits(floor: Dict[str, float]) -> Dict[str, float]: | |
| return dict(one_minus_pcc=max(FLOOR_X * floor["one_minus_pcc"], FLOOR_MIN), | |
| **{k: max(GRP_X * floor[k], GRP_MIN[k]) for k in ("pos_m", "rot", "grip_m")}) | |
| def judge(got: Dict[str, float], floor: Dict[str, float]) -> Dict[str, object]: | |
| """Floor-anchored A2 verdict of one case: groups first (primary), then PCC; ratios to the floor.""" | |
| lim = limits(floor) | |
| ok_g = all(got[k] <= lim[k] for k in ("pos_m", "rot", "grip_m")) | |
| ok_p = got["one_minus_pcc"] <= lim["one_minus_pcc"] | |
| ratio = {k: got[k] / max(floor[k], 1e-12) for k in ("one_minus_pcc", "pos_m", "rot", "grip_m")} | |
| return dict(limits=lim, groups_pass=ok_g, pcc_pass=ok_p, passed=ok_g and ok_p, ratio_to_floor=ratio) | |