Download training/bonsai_hf.py from KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA: direct link, hf CLI and curl.
- Browser
- Download file 15.8 kB
-
https://huggingface.co/KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA/resolve/main/training/bonsai_hf.py
- Command line
-
hf download hf://KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA/training/bonsai_hf.py
-
curl -L -o bonsai_hf.py https://huggingface.co/KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA/resolve/main/training/bonsai_hf.py
15.8 kB
| """Ternary-Bonsai-2-9B (PQ2_0 + Hadamard) rebuilt exactly in PyTorch on top of transformers' Qwen3_5TextModel, to train | |
| LoRA adapters on a GTX 1080. Ternary weights stay packed on the GPU (2 bit codes + one fp16 scale per 128 weights) and are | |
| unpacked to fp32 at each use. The input is rotated as in llama.cpp (build_lora_mm): x' = H(s * x), y = W' x'. The LoRA reads | |
| the input BEFORE the rotation, so the adapter is the same GGUF LoRA the PrismML fork applies with --lora. | |
| Everything is computed in fp32 (fp16 runs at 1/64 speed on Pascal). | |
| HF ordering: V rows of qkv/z/a/b, conv1d channels, dt and A are grouped in HF and interleaved in the GGUF (the converter | |
| interleaves them); ssm_out already has grouped columns in the GGUF, so it needs no permutation. | |
| Paths (environment variables): | |
| BONSAI_GGUF Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf (or PQ2_0) from prism-ml/Ternary-Bonsai-2-9B-gguf | |
| QWEN_SRC folder with config.json from Qwen/Qwen3.5-9B (same architecture, no weights needed) | |
| GGUF_PY gguf-py folder of the PrismML llama.cpp fork (it knows the PQ2_0 type) | |
| """ | |
| import json | |
| import math | |
| import os | |
| import sys | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| sys.path.insert(0, os.environ.get("GGUF_PY", "llama.cpp/gguf-py")) | |
| import gguf # noqa: E402 | |
| from transformers.models.qwen3_5 import modeling_qwen3_5 as mq # noqa: E402 | |
| from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5Config # noqa: E402 | |
| GGUF = os.environ.get("BONSAI_GGUF", "Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf") | |
| CONFIG = os.path.join(os.environ.get("QWEN_SRC", "qwen35-9b"), "config.json") | |
| MAPPA = {"linear_attn.in_proj_qkv": "attn_qkv", "linear_attn.in_proj_z": "attn_gate", "linear_attn.out_proj": "ssm_out", | |
| "self_attn.q_proj": "attn_q", "self_attn.k_proj": "attn_k", "self_attn.v_proj": "attn_v", "self_attn.o_proj": "attn_output", | |
| "mlp.gate_proj": "ffn_gate", "mlp.up_proj": "ffn_up", "mlp.down_proj": "ffn_down"} | |
| def hadamard(n, device): | |
| i = torch.arange(n, device=device) | |
| par = torch.zeros(n, n, dtype=torch.int64, device=device) | |
| x = i[:, None] & i[None, :] | |
| while bool(x.any()): | |
| par ^= x & 1 | |
| x = x >> 1 | |
| return (1 - 2 * par).float() / math.sqrt(n) | |
| try: # own CUDA kernel (tern_unpack.py, compiled once): writes W*scale in one pass; without it, plain torch fallback | |
| import tern_unpack | |
| _KERNEL = tern_unpack._carica() | |
| except Exception: | |
| _KERNEL = None | |
| def spacchetta(codici, scale): # codes (out, in/4) uint8, scales (out, in/128) fp16 -> (out, in) fp32 | |
| if _KERNEL is not None and codici.is_cuda and codici.is_contiguous() and scale.is_contiguous(): | |
| return _KERNEL.unpack(codici, scale, torch.empty(codici.shape[0], codici.shape[1] * 4, device=codici.device)) | |
| # broadcast shift and (q-1)*scale in place on an (out, in/128, 128) view: same values, half the memory traffic | |
| s = torch.tensor([0, 2, 4, 6], dtype=torch.uint8, device=codici.device) | |
| w = ((codici.unsqueeze(-1) >> s) & 3).float().view(codici.shape[0], -1, 128) | |
| return w.sub_(1.0).mul_(scale.float().unsqueeze(-1)).view(codici.shape[0], -1) | |
| def kron2(z, Hs): # z (..., a*a) @ kron(Hs, Hs): Sylvester H4096 = H64 x H64, two 64-wide matmuls instead of one dense 4096 | |
| a = Hs.shape[0] | |
| t = z.reshape(-1, a) @ Hs # low index | |
| return torch.matmul(Hs, t.view(-1, a, a)).reshape(z.shape) # high index | |
| class _MatTernaria(torch.autograd.Function): # y = x W^T without saving W: unpacked again in backward | |
| def forward(ctx, x, codici, scale): | |
| ctx.save_for_backward(codici, scale) | |
| return x @ spacchetta(codici, scale).T | |
| def backward(ctx, gy): | |
| codici, scale = ctx.saved_tensors | |
| return gy @ spacchetta(codici, scale), None, None | |
| class Ruota(nn.Module): | |
| def __init__(self, segni, H): | |
| super().__init__() | |
| self.register_buffer("segni", segni, persistent=False) | |
| self.H = H # shared, not registered | |
| def forward(self, x): | |
| # q/k/v, gate/up and qkv/z get the same tensor: rotate once (like the fork's memo) | |
| if getattr(self, "_x", None) is x and self._g == torch.is_grad_enabled(): | |
| return self._y | |
| self._x, self._g, self._y = x, torch.is_grad_enabled(), kron2(x * self.segni, self.H) | |
| return self._y | |
| class LinearTernario(nn.Module): | |
| def __init__(self, codici, scale, ruota, rank=0, alpha=None): | |
| super().__init__() | |
| self.register_buffer("codici", codici, persistent=False) | |
| self.register_buffer("scale", scale, persistent=False) | |
| self.ruota = ruota | |
| self.out_f, self.in_f = codici.shape[0], codici.shape[1] * 4 | |
| self.rank = rank | |
| if rank: | |
| self.lora_a = nn.Parameter(torch.randn(rank, self.in_f, device=codici.device) / math.sqrt(self.in_f)) | |
| self.lora_b = nn.Parameter(torch.zeros(self.out_f, rank, device=codici.device)) | |
| self.s = (alpha or rank) / rank | |
| def forward(self, x): | |
| if self.out_f > 65536: # 248k x 4096 output head: unpacked in slices to fit in VRAM | |
| xr = self.ruota(x.float()) | |
| return torch.cat([_MatTernaria.apply(xr, self.codici[i:i + 16384], self.scale[i:i + 16384]) | |
| for i in range(0, self.out_f, 16384)], -1) | |
| y = _MatTernaria.apply(self.ruota(x.float()), self.codici, self.scale) | |
| if self.rank: | |
| y = y + (x.float() @ self.lora_a.T) @ self.lora_b.T * self.s | |
| return y | |
| class EmbedTernario(nn.Module): # rotated row: h = s * (H z), like build_embd_rows | |
| def __init__(self, codici, scale, segni, H): | |
| super().__init__() | |
| self.register_buffer("codici", codici, persistent=False) | |
| self.register_buffer("scale", scale, persistent=False) | |
| self.register_buffer("segni", segni, persistent=False) | |
| self.H = H | |
| def forward(self, ids): | |
| z = spacchetta(self.codici[ids.reshape(-1)], self.scale[ids.reshape(-1)]) | |
| h = kron2(z, self.H) * self.segni | |
| return h.reshape(*ids.shape, -1) | |
| def _gruppi(t, dim, nk, nvpk, hd): # inverse of _reorder_v_heads: interleaved -> grouped | |
| s = list(t.shape) | |
| t = t.reshape(*s[:dim], nvpk, nk, hd, *s[dim + 1:]) | |
| p = list(range(t.ndim)) | |
| p[dim], p[dim + 1] = p[dim + 1], p[dim] | |
| return t.permute(*p).contiguous().reshape(*s) | |
| def carica(device="cuda", rank=0, alpha=None, moduli=tuple(MAPPA), gguf_path=GGUF): | |
| """Returns (model, output_head, config). Only the LoRA tensors require grad.""" | |
| cfg = Qwen3_5Config(**{k: v for k, v in json.load(open(CONFIG)).items()}).text_config | |
| cfg._attn_implementation = "sdpa" | |
| r = gguf.GGUFReader(gguf_path) | |
| T = {t.name: t for t in r.tensors} | |
| sv = r.fields["prism.hadamard.sign_values"].contents() | |
| sw = r.fields["prism.hadamard.sign_widths"].contents() | |
| segni, o = {}, 0 | |
| for w in sw: | |
| segni[w] = torch.tensor(sv[o:o + w], dtype=torch.float32, device=device) | |
| o += w | |
| H = hadamard(math.isqrt(int(r.fields["prism.hadamard.block_size"].contents())), device) # a x a factor, block_size = a*a | |
| ruote = {w: Ruota(s, H) for w, s in segni.items()} | |
| nk, nv, hd = cfg.linear_num_key_heads, cfg.linear_num_value_heads, cfg.linear_value_head_dim | |
| def f32(nome): | |
| t = T[nome] | |
| return torch.from_numpy(np.array(t.data, dtype=np.float32).reshape([int(x) for x in reversed(t.shape)])) | |
| def tern(nome, gruppi_righe=False): | |
| t = T[nome] | |
| assert t.tensor_type.name == "PQ2_0", nome | |
| out, inn = int(t.shape[1]), int(t.shape[0]) | |
| b = np.frombuffer(t.data.tobytes(), dtype=np.uint8).reshape(out, inn // 128, 34) | |
| sc = torch.from_numpy(b[:, :, :2].copy().view(np.float16).reshape(out, inn // 128)) | |
| cod = torch.from_numpy(b[:, :, 2:].copy().reshape(out, inn // 4)) | |
| if gruppi_righe == "qkv": | |
| q = 2 * cfg.linear_key_head_dim * nk | |
| cod = torch.cat([cod[:q], _gruppi(cod[q:], 0, nk, nv // nk, hd)]) | |
| sc = torch.cat([sc[:q], _gruppi(sc[q:], 0, nk, nv // nk, hd)]) | |
| elif gruppi_righe == "z": | |
| cod, sc = _gruppi(cod, 0, nk, nv // nk, hd), _gruppi(sc, 0, nk, nv // nk, hd) | |
| return cod.to(device), sc.to(device), inn | |
| with torch.device("meta"): | |
| m = mq.Qwen3_5TextModel(cfg) | |
| for i, L in enumerate(m.layers): | |
| for hf, gg in MAPPA.items(): | |
| parent, attr = hf.split(".") | |
| mod = getattr(L, parent, None) | |
| if mod is None or not hasattr(mod, attr): | |
| continue | |
| cod, sc, inn = tern(f"blk.{i}.{gg}.weight", "qkv" if gg == "attn_qkv" else "z" if gg == "attn_gate" else False) | |
| setattr(mod, attr, LinearTernario(cod, sc, ruote[inn], rank if hf in moduli else 0, alpha)) | |
| p = f"blk.{i}." | |
| L.input_layernorm.weight = nn.Parameter(f32(p + "attn_norm.weight").to(device) - 1, requires_grad=False) | |
| L.post_attention_layernorm.weight = nn.Parameter(f32(p + "post_attention_norm.weight").to(device) - 1, requires_grad=False) | |
| if hasattr(L, "linear_attn"): | |
| g = L.linear_attn | |
| g.A_log = nn.Parameter(torch.log(-_gruppi(f32(p + "ssm_a").reshape(-1, 1), 0, nk, nv // nk, 1).reshape(-1)).to(device), requires_grad=False) | |
| g.dt_bias = nn.Parameter(_gruppi(f32(p + "ssm_dt.bias").reshape(-1, 1), 0, nk, nv // nk, 1).reshape(-1).to(device), requires_grad=False) | |
| c = f32(p + "ssm_conv1d.weight") # (channels, 4) | |
| qk = 2 * cfg.linear_key_head_dim * nk | |
| c = torch.cat([c[:qk], _gruppi(c[qk:], 0, nk, nv // nk, hd)]) | |
| g.conv1d.weight = nn.Parameter(c.unsqueeze(1).to(device), requires_grad=False) | |
| for hf, gg in (("in_proj_a", "ssm_alpha"), ("in_proj_b", "ssm_beta")): | |
| w = _gruppi(f32(p + gg + ".weight"), 0, nk, nv // nk, 1) | |
| lin = nn.Linear(w.shape[1], w.shape[0], bias=False, device=device) | |
| lin.weight = nn.Parameter(w.to(device), requires_grad=False) | |
| setattr(g, hf, lin) | |
| g.norm.weight = nn.Parameter(f32(p + "ssm_norm.weight").to(device), requires_grad=False) | |
| else: | |
| a = L.self_attn | |
| a.q_norm.weight = nn.Parameter(f32(p + "attn_q_norm.weight").to(device) - 1, requires_grad=False) | |
| a.k_norm.weight = nn.Parameter(f32(p + "attn_k_norm.weight").to(device) - 1, requires_grad=False) | |
| m.norm.weight = nn.Parameter(f32("output_norm.weight").to(device) - 1, requires_grad=False) | |
| cod, sc, inn = tern("token_embd.weight") | |
| m.embed_tokens = EmbedTernario(cod, sc, segni[inn], H) | |
| cod, sc, inn = tern("output.weight") | |
| testa = LinearTernario(cod, sc, ruote[inn]) | |
| m.rotary_emb = mq.Qwen3_5TextRotaryEmbedding(config=cfg, device=device) | |
| for n, b in list(m.named_buffers()): | |
| assert b.device.type != "meta", n | |
| for n, p in m.named_parameters(): | |
| assert p.device.type != "meta", n | |
| p.requires_grad = "lora_" in n | |
| return m, testa, cfg | |
| def salva_lora_gguf(m, path, rank, alpha=None): # same format the fork reads with --lora | |
| w = gguf.GGUFWriter(path, "qwen35") | |
| w.add_type(gguf.GGUFType.ADAPTER) | |
| w.add_string(gguf.Keys.Adapter.TYPE, "lora") | |
| w.add_float32(gguf.Keys.Adapter.LORA_ALPHA, float(alpha or rank)) | |
| for i, L in enumerate(m.layers): | |
| for hf, gg in MAPPA.items(): | |
| parent, attr = hf.split(".") | |
| mod = getattr(getattr(L, parent, None), attr, None) | |
| if not isinstance(mod, LinearTernario) or not mod.rank: | |
| continue | |
| a, b = mod.lora_a.detach().float().cpu(), mod.lora_b.detach().float().cpu() | |
| if gg == "attn_qkv": # V rows interleaved again as in the base GGUF | |
| q = 2 * 128 * 16 | |
| b = torch.cat([b[:q], _affianca(b[q:])]) | |
| elif gg == "attn_gate": | |
| b = _affianca(b) | |
| w.add_tensor(f"blk.{i}.{gg}.weight.lora_a", a.to(torch.float16).numpy()) | |
| w.add_tensor(f"blk.{i}.{gg}.weight.lora_b", b.to(torch.float16).numpy()) | |
| w.write_header_to_file() | |
| w.write_kv_data_to_file() | |
| w.write_tensors_to_file() | |
| w.close() | |
| def _affianca(t, nk=16, nvpk=2, hd=128): | |
| s = list(t.shape) | |
| t = t.reshape(nk, nvpk, hd, *s[1:]) | |
| return t.permute(1, 0, 2, *range(3, t.ndim)).contiguous().reshape(s) | |
| # ---- Shared prefix. K targets on the same screenshot = one sequence X = [prefix | suffix 1 | ... | suffix K] (lens = lengths). | |
| # Projections, norms and MLP run on all tokens together (one weight unpack); causal conv, GDN recurrence and attention run per | |
| # segment: the prefix alone, every branch with the prefix's state / keys. Same math as Qwen3_5DecoderLayer on each full | |
| # sequence (different summation order: within tolerance, not bit-exact). | |
| def _gdn(g, x, lens): | |
| n, k = x.shape[1], g.conv_kernel_size | |
| mixed = g.in_proj_qkv(x).transpose(1, 2) # (1, C, T) | |
| z = g.in_proj_z(x).reshape(1, n, -1, g.head_v_dim) | |
| beta, a = g.in_proj_b(x).sigmoid(), g.in_proj_a(x) | |
| seg = mixed.split(lens, -1) | |
| w, ctx = g.conv1d.weight, seg[0][..., -(k - 1):] # branches see the last k-1 REAL inputs of the prefix, not zeros | |
| conv = lambda s: F.conv1d(s, w, None, groups=w.shape[0]) # noqa: E731 | |
| mixed = torch.cat([conv(F.pad(seg[0], (k - 1, 0)))] + [conv(torch.cat([ctx, s], -1)) for s in seg[1:]], -1) | |
| mixed = mq.ACT2FN[g.activation](mixed).transpose(1, 2) | |
| q, kk, v = torch.split(mixed, [g.key_dim, g.key_dim, g.value_dim], -1) | |
| q, kk, v = (t.reshape(1, n, -1, d) for t, d in ((q, g.head_k_dim), (kk, g.head_k_dim), (v, g.head_v_dim))) | |
| rip = g.num_v_heads // g.num_k_heads | |
| if rip > 1: | |
| q, kk = q.repeat_interleave(rip, 2), kk.repeat_interleave(rip, 2) | |
| dec = -g.A_log.float().exp() * F.softplus(a.float() + g.dt_bias) | |
| out, stato = [], None | |
| for i, (qi, ki, vi, di, bi) in enumerate(zip(*(t.split(lens, 1) for t in (q, kk, v, dec, beta)))): | |
| o, s = mq.torch_chunk_gated_delta_rule(qi, ki, vi, g=di, beta=bi, initial_state=stato, output_final_state=i == 0, | |
| use_qk_l2norm_in_kernel=True) | |
| out.append(o) | |
| stato = s if i == 0 else stato # state after the prefix (padding to 64 uses k=v=beta=0, g=0, so it stays unchanged) | |
| o = g.norm(torch.cat(out, 1).reshape(-1, g.head_v_dim), z.reshape(-1, g.head_v_dim)).reshape(1, n, -1) | |
| return g.out_proj(o) | |
| def _attn(at, x, cos, sin, lens): | |
| n = x.shape[1] | |
| sh = (1, n, -1, at.head_dim) | |
| q, gate = torch.chunk(at.q_proj(x).view(1, n, -1, at.head_dim * 2), 2, -1) | |
| q, k = mq.apply_rotary_pos_emb(at.q_norm(q.reshape(sh)).transpose(1, 2), at.k_norm(at.k_proj(x).view(sh)).transpose(1, 2), cos, sin) | |
| v = at.v_proj(x).view(sh).transpose(1, 2) | |
| qs, ks, vs = q.split(lens, 2), k.split(lens, 2), v.split(lens, 2) | |
| f = mq.ALL_ATTENTION_FUNCTIONS.get_interface(at.config._attn_implementation, mq.eager_attention_forward) | |
| out = [f(at, qs[0], ks[0], vs[0], None, dropout=0.0, scaling=at.scaling)[0]] # prefix: causal | |
| for qi, ki, vi in zip(qs[1:], ks[1:], vs[1:]): # branch: keys = prefix + own suffix, each query sees the prefix and its past | |
| kk, vv = torch.cat([ks[0], ki], 2), torch.cat([vs[0], vi], 2) | |
| mask = torch.ones(qi.shape[2], kk.shape[2], dtype=torch.bool, device=x.device).tril(lens[0])[None, None] | |
| out.append(f(at, qi, kk, vv, mask, dropout=0.0, scaling=at.scaling)[0]) | |
| return at.o_proj(torch.cat(out, 1).reshape(1, n, -1) * torch.sigmoid(gate.reshape(1, n, -1))) | |
| def strato_ramificato(L, X, cos, sin, lens): | |
| h = L.input_layernorm(X) | |
| X = X + (_gdn(L.linear_attn, h, lens) if L.block_type == "linear_attention" else _attn(L.self_attn, h, cos, sin, lens)) | |
| return X + L.mlp(L.post_attention_layernorm(X)) | |