"""Blackwell mode for bonsai_hf: unpack every ternary weight ONCE (fp32, ~36 GB on a 96 GB card) instead of at each use, and let matmuls run in TF32, or with BF16=1 keep the frozen weights in bf16 (LoRA, rotation and loss stay fp32). Same rotation, same LoRA placement, same adapter format as the 1080 kit. import bonsai_hf as B, bonsai_bw m, testa, cfg = B.carica(rank=R, alpha=A) bonsai_bw.densifica(m, testa) """ import os import torch import bonsai_hf as B torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True BF16 = os.environ.get("BF16") == "1" # frozen base matmuls in bf16 (386 TFLOPS measured on this card, TF32 201) def _fwd_lineare(self, x): xr = self.ruota(x.float()) y = (xr.to(self.W.dtype) @ self.W.T).float() if self.rank: y = y + (x.float() @ self.lora_a.T) @ self.lora_b.T * self.s return y def _fwd_embed(self, ids): return self.W[ids].float() @torch.no_grad() def densifica(*moduli): n = 0 for root in moduli: for mod in root.modules(): if isinstance(mod, B.LinearTernario) and not hasattr(mod, "W"): W = B.spacchetta(mod.codici, mod.scale) mod.register_buffer("W", W.bfloat16() if BF16 else W, persistent=False) del W mod.forward = _fwd_lineare.__get__(mod) del mod.codici, mod.scale n += 1 elif isinstance(mod, B.EmbedTernario) and not hasattr(mod, "W"): W = torch.empty(mod.codici.shape[0], mod.codici.shape[1] * 4, device=mod.codici.device) for i in range(0, W.shape[0], 16384): # rows already rotated and signed, like EmbedTernario.forward z = B.spacchetta(mod.codici[i:i + 16384], mod.scale[i:i + 16384]) W[i:i + 16384] = B.kron2(z, mod.H) * mod.segni mod.register_buffer("W", W, persistent=False) mod.forward = _fwd_embed.__get__(mod) del mod.codici, mod.scale n += 1 torch.cuda.empty_cache() print(f"densified {n} ternary modules, {torch.cuda.memory_allocated() / 2**30:.1f} GiB allocated", flush=True) if __name__ == "__main__": # self-check: dense output == packed output on a few rows m, testa, _ = B.carica(rank=0) lin = next(x for x in m.modules() if isinstance(x, B.LinearTernario)) x = torch.randn(3, lin.in_f, device="cuda") torch.backends.cuda.matmul.allow_tf32 = False ref = lin(x) densifica(lin) out = lin(x) print("max abs diff fp32 dense vs packed:", float((out - ref).abs().max())) assert torch.allclose(out, ref, atol=1e-4, rtol=1e-4)