adriandj3's picture
TernaryPilot v2: card and training kit (weights after training)
5c251fb verified
Raw History Blame Contribute Delete
2.68 kB
"""Blackwell mode for bonsai_hf: unpack every ternary weight ONCE (fp32, ~36 GB on a 96 GB card) instead of at each use,
and let matmuls run in TF32, or with BF16=1 keep the frozen weights in bf16 (LoRA, rotation and loss stay fp32). Same rotation, same LoRA placement, same adapter format as the 1080 kit.
import bonsai_hf as B, bonsai_bw
m, testa, cfg = B.carica(rank=R, alpha=A)
bonsai_bw.densifica(m, testa)
"""
import os
import torch
import bonsai_hf as B
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
BF16 = os.environ.get("BF16") == "1" # frozen base matmuls in bf16 (386 TFLOPS measured on this card, TF32 201)
def _fwd_lineare(self, x):
xr = self.ruota(x.float())
y = (xr.to(self.W.dtype) @ self.W.T).float()
if self.rank:
y = y + (x.float() @ self.lora_a.T) @ self.lora_b.T * self.s
return y
def _fwd_embed(self, ids):
return self.W[ids].float()
@torch.no_grad()
def densifica(*moduli):
n = 0
for root in moduli:
for mod in root.modules():
if isinstance(mod, B.LinearTernario) and not hasattr(mod, "W"):
W = B.spacchetta(mod.codici, mod.scale)
mod.register_buffer("W", W.bfloat16() if BF16 else W, persistent=False)
del W
mod.forward = _fwd_lineare.__get__(mod)
del mod.codici, mod.scale
n += 1
elif isinstance(mod, B.EmbedTernario) and not hasattr(mod, "W"):
W = torch.empty(mod.codici.shape[0], mod.codici.shape[1] * 4, device=mod.codici.device)
for i in range(0, W.shape[0], 16384): # rows already rotated and signed, like EmbedTernario.forward
z = B.spacchetta(mod.codici[i:i + 16384], mod.scale[i:i + 16384])
W[i:i + 16384] = B.kron2(z, mod.H) * mod.segni
mod.register_buffer("W", W, persistent=False)
mod.forward = _fwd_embed.__get__(mod)
del mod.codici, mod.scale
n += 1
torch.cuda.empty_cache()
print(f"densified {n} ternary modules, {torch.cuda.memory_allocated() / 2**30:.1f} GiB allocated", flush=True)
if __name__ == "__main__": # self-check: dense output == packed output on a few rows
m, testa, _ = B.carica(rank=0)
lin = next(x for x in m.modules() if isinstance(x, B.LinearTernario))
x = torch.randn(3, lin.in_f, device="cuda")
torch.backends.cuda.matmul.allow_tf32 = False
ref = lin(x)
densifica(lin)
out = lin(x)
print("max abs diff fp32 dense vs packed:", float((out - ref).abs().max()))
assert torch.allclose(out, ref, atol=1e-4, rtol=1e-4)