adriandj3's picture
TernaryPilot v2: card and training kit (weights after training)
5c251fb verified
Raw History Blame Contribute Delete
15.8 kB
"""Ternary-Bonsai-2-9B (PQ2_0 + Hadamard) rebuilt exactly in PyTorch on top of transformers' Qwen3_5TextModel, to train
LoRA adapters on a GTX 1080. Ternary weights stay packed on the GPU (2 bit codes + one fp16 scale per 128 weights) and are
unpacked to fp32 at each use. The input is rotated as in llama.cpp (build_lora_mm): x' = H(s * x), y = W' x'. The LoRA reads
the input BEFORE the rotation, so the adapter is the same GGUF LoRA the PrismML fork applies with --lora.
Everything is computed in fp32 (fp16 runs at 1/64 speed on Pascal).
HF ordering: V rows of qkv/z/a/b, conv1d channels, dt and A are grouped in HF and interleaved in the GGUF (the converter
interleaves them); ssm_out already has grouped columns in the GGUF, so it needs no permutation.
Paths (environment variables):
BONSAI_GGUF Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf (or PQ2_0) from prism-ml/Ternary-Bonsai-2-9B-gguf
QWEN_SRC folder with config.json from Qwen/Qwen3.5-9B (same architecture, no weights needed)
GGUF_PY gguf-py folder of the PrismML llama.cpp fork (it knows the PQ2_0 type)
"""
import json
import math
import os
import sys
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
sys.path.insert(0, os.environ.get("GGUF_PY", "llama.cpp/gguf-py"))
import gguf # noqa: E402
from transformers.models.qwen3_5 import modeling_qwen3_5 as mq # noqa: E402
from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5Config # noqa: E402
GGUF = os.environ.get("BONSAI_GGUF", "Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf")
CONFIG = os.path.join(os.environ.get("QWEN_SRC", "qwen35-9b"), "config.json")
MAPPA = {"linear_attn.in_proj_qkv": "attn_qkv", "linear_attn.in_proj_z": "attn_gate", "linear_attn.out_proj": "ssm_out",
"self_attn.q_proj": "attn_q", "self_attn.k_proj": "attn_k", "self_attn.v_proj": "attn_v", "self_attn.o_proj": "attn_output",
"mlp.gate_proj": "ffn_gate", "mlp.up_proj": "ffn_up", "mlp.down_proj": "ffn_down"}
def hadamard(n, device):
i = torch.arange(n, device=device)
par = torch.zeros(n, n, dtype=torch.int64, device=device)
x = i[:, None] & i[None, :]
while bool(x.any()):
par ^= x & 1
x = x >> 1
return (1 - 2 * par).float() / math.sqrt(n)
try: # own CUDA kernel (tern_unpack.py, compiled once): writes W*scale in one pass; without it, plain torch fallback
import tern_unpack
_KERNEL = tern_unpack._carica()
except Exception:
_KERNEL = None
def spacchetta(codici, scale): # codes (out, in/4) uint8, scales (out, in/128) fp16 -> (out, in) fp32
if _KERNEL is not None and codici.is_cuda and codici.is_contiguous() and scale.is_contiguous():
return _KERNEL.unpack(codici, scale, torch.empty(codici.shape[0], codici.shape[1] * 4, device=codici.device))
# broadcast shift and (q-1)*scale in place on an (out, in/128, 128) view: same values, half the memory traffic
s = torch.tensor([0, 2, 4, 6], dtype=torch.uint8, device=codici.device)
w = ((codici.unsqueeze(-1) >> s) & 3).float().view(codici.shape[0], -1, 128)
return w.sub_(1.0).mul_(scale.float().unsqueeze(-1)).view(codici.shape[0], -1)
def kron2(z, Hs): # z (..., a*a) @ kron(Hs, Hs): Sylvester H4096 = H64 x H64, two 64-wide matmuls instead of one dense 4096
a = Hs.shape[0]
t = z.reshape(-1, a) @ Hs # low index
return torch.matmul(Hs, t.view(-1, a, a)).reshape(z.shape) # high index
class _MatTernaria(torch.autograd.Function): # y = x W^T without saving W: unpacked again in backward
@staticmethod
def forward(ctx, x, codici, scale):
ctx.save_for_backward(codici, scale)
return x @ spacchetta(codici, scale).T
@staticmethod
def backward(ctx, gy):
codici, scale = ctx.saved_tensors
return gy @ spacchetta(codici, scale), None, None
class Ruota(nn.Module):
def __init__(self, segni, H):
super().__init__()
self.register_buffer("segni", segni, persistent=False)
self.H = H # shared, not registered
def forward(self, x):
# q/k/v, gate/up and qkv/z get the same tensor: rotate once (like the fork's memo)
if getattr(self, "_x", None) is x and self._g == torch.is_grad_enabled():
return self._y
self._x, self._g, self._y = x, torch.is_grad_enabled(), kron2(x * self.segni, self.H)
return self._y
class LinearTernario(nn.Module):
def __init__(self, codici, scale, ruota, rank=0, alpha=None):
super().__init__()
self.register_buffer("codici", codici, persistent=False)
self.register_buffer("scale", scale, persistent=False)
self.ruota = ruota
self.out_f, self.in_f = codici.shape[0], codici.shape[1] * 4
self.rank = rank
if rank:
self.lora_a = nn.Parameter(torch.randn(rank, self.in_f, device=codici.device) / math.sqrt(self.in_f))
self.lora_b = nn.Parameter(torch.zeros(self.out_f, rank, device=codici.device))
self.s = (alpha or rank) / rank
def forward(self, x):
if self.out_f > 65536: # 248k x 4096 output head: unpacked in slices to fit in VRAM
xr = self.ruota(x.float())
return torch.cat([_MatTernaria.apply(xr, self.codici[i:i + 16384], self.scale[i:i + 16384])
for i in range(0, self.out_f, 16384)], -1)
y = _MatTernaria.apply(self.ruota(x.float()), self.codici, self.scale)
if self.rank:
y = y + (x.float() @ self.lora_a.T) @ self.lora_b.T * self.s
return y
class EmbedTernario(nn.Module): # rotated row: h = s * (H z), like build_embd_rows
def __init__(self, codici, scale, segni, H):
super().__init__()
self.register_buffer("codici", codici, persistent=False)
self.register_buffer("scale", scale, persistent=False)
self.register_buffer("segni", segni, persistent=False)
self.H = H
def forward(self, ids):
z = spacchetta(self.codici[ids.reshape(-1)], self.scale[ids.reshape(-1)])
h = kron2(z, self.H) * self.segni
return h.reshape(*ids.shape, -1)
def _gruppi(t, dim, nk, nvpk, hd): # inverse of _reorder_v_heads: interleaved -> grouped
s = list(t.shape)
t = t.reshape(*s[:dim], nvpk, nk, hd, *s[dim + 1:])
p = list(range(t.ndim))
p[dim], p[dim + 1] = p[dim + 1], p[dim]
return t.permute(*p).contiguous().reshape(*s)
def carica(device="cuda", rank=0, alpha=None, moduli=tuple(MAPPA), gguf_path=GGUF):
"""Returns (model, output_head, config). Only the LoRA tensors require grad."""
cfg = Qwen3_5Config(**{k: v for k, v in json.load(open(CONFIG)).items()}).text_config
cfg._attn_implementation = "sdpa"
r = gguf.GGUFReader(gguf_path)
T = {t.name: t for t in r.tensors}
sv = r.fields["prism.hadamard.sign_values"].contents()
sw = r.fields["prism.hadamard.sign_widths"].contents()
segni, o = {}, 0
for w in sw:
segni[w] = torch.tensor(sv[o:o + w], dtype=torch.float32, device=device)
o += w
H = hadamard(math.isqrt(int(r.fields["prism.hadamard.block_size"].contents())), device) # a x a factor, block_size = a*a
ruote = {w: Ruota(s, H) for w, s in segni.items()}
nk, nv, hd = cfg.linear_num_key_heads, cfg.linear_num_value_heads, cfg.linear_value_head_dim
def f32(nome):
t = T[nome]
return torch.from_numpy(np.array(t.data, dtype=np.float32).reshape([int(x) for x in reversed(t.shape)]))
def tern(nome, gruppi_righe=False):
t = T[nome]
assert t.tensor_type.name == "PQ2_0", nome
out, inn = int(t.shape[1]), int(t.shape[0])
b = np.frombuffer(t.data.tobytes(), dtype=np.uint8).reshape(out, inn // 128, 34)
sc = torch.from_numpy(b[:, :, :2].copy().view(np.float16).reshape(out, inn // 128))
cod = torch.from_numpy(b[:, :, 2:].copy().reshape(out, inn // 4))
if gruppi_righe == "qkv":
q = 2 * cfg.linear_key_head_dim * nk
cod = torch.cat([cod[:q], _gruppi(cod[q:], 0, nk, nv // nk, hd)])
sc = torch.cat([sc[:q], _gruppi(sc[q:], 0, nk, nv // nk, hd)])
elif gruppi_righe == "z":
cod, sc = _gruppi(cod, 0, nk, nv // nk, hd), _gruppi(sc, 0, nk, nv // nk, hd)
return cod.to(device), sc.to(device), inn
with torch.device("meta"):
m = mq.Qwen3_5TextModel(cfg)
for i, L in enumerate(m.layers):
for hf, gg in MAPPA.items():
parent, attr = hf.split(".")
mod = getattr(L, parent, None)
if mod is None or not hasattr(mod, attr):
continue
cod, sc, inn = tern(f"blk.{i}.{gg}.weight", "qkv" if gg == "attn_qkv" else "z" if gg == "attn_gate" else False)
setattr(mod, attr, LinearTernario(cod, sc, ruote[inn], rank if hf in moduli else 0, alpha))
p = f"blk.{i}."
L.input_layernorm.weight = nn.Parameter(f32(p + "attn_norm.weight").to(device) - 1, requires_grad=False)
L.post_attention_layernorm.weight = nn.Parameter(f32(p + "post_attention_norm.weight").to(device) - 1, requires_grad=False)
if hasattr(L, "linear_attn"):
g = L.linear_attn
g.A_log = nn.Parameter(torch.log(-_gruppi(f32(p + "ssm_a").reshape(-1, 1), 0, nk, nv // nk, 1).reshape(-1)).to(device), requires_grad=False)
g.dt_bias = nn.Parameter(_gruppi(f32(p + "ssm_dt.bias").reshape(-1, 1), 0, nk, nv // nk, 1).reshape(-1).to(device), requires_grad=False)
c = f32(p + "ssm_conv1d.weight") # (channels, 4)
qk = 2 * cfg.linear_key_head_dim * nk
c = torch.cat([c[:qk], _gruppi(c[qk:], 0, nk, nv // nk, hd)])
g.conv1d.weight = nn.Parameter(c.unsqueeze(1).to(device), requires_grad=False)
for hf, gg in (("in_proj_a", "ssm_alpha"), ("in_proj_b", "ssm_beta")):
w = _gruppi(f32(p + gg + ".weight"), 0, nk, nv // nk, 1)
lin = nn.Linear(w.shape[1], w.shape[0], bias=False, device=device)
lin.weight = nn.Parameter(w.to(device), requires_grad=False)
setattr(g, hf, lin)
g.norm.weight = nn.Parameter(f32(p + "ssm_norm.weight").to(device), requires_grad=False)
else:
a = L.self_attn
a.q_norm.weight = nn.Parameter(f32(p + "attn_q_norm.weight").to(device) - 1, requires_grad=False)
a.k_norm.weight = nn.Parameter(f32(p + "attn_k_norm.weight").to(device) - 1, requires_grad=False)
m.norm.weight = nn.Parameter(f32("output_norm.weight").to(device) - 1, requires_grad=False)
cod, sc, inn = tern("token_embd.weight")
m.embed_tokens = EmbedTernario(cod, sc, segni[inn], H)
cod, sc, inn = tern("output.weight")
testa = LinearTernario(cod, sc, ruote[inn])
m.rotary_emb = mq.Qwen3_5TextRotaryEmbedding(config=cfg, device=device)
for n, b in list(m.named_buffers()):
assert b.device.type != "meta", n
for n, p in m.named_parameters():
assert p.device.type != "meta", n
p.requires_grad = "lora_" in n
return m, testa, cfg
def salva_lora_gguf(m, path, rank, alpha=None): # same format the fork reads with --lora
w = gguf.GGUFWriter(path, "qwen35")
w.add_type(gguf.GGUFType.ADAPTER)
w.add_string(gguf.Keys.Adapter.TYPE, "lora")
w.add_float32(gguf.Keys.Adapter.LORA_ALPHA, float(alpha or rank))
for i, L in enumerate(m.layers):
for hf, gg in MAPPA.items():
parent, attr = hf.split(".")
mod = getattr(getattr(L, parent, None), attr, None)
if not isinstance(mod, LinearTernario) or not mod.rank:
continue
a, b = mod.lora_a.detach().float().cpu(), mod.lora_b.detach().float().cpu()
if gg == "attn_qkv": # V rows interleaved again as in the base GGUF
q = 2 * 128 * 16
b = torch.cat([b[:q], _affianca(b[q:])])
elif gg == "attn_gate":
b = _affianca(b)
w.add_tensor(f"blk.{i}.{gg}.weight.lora_a", a.to(torch.float16).numpy())
w.add_tensor(f"blk.{i}.{gg}.weight.lora_b", b.to(torch.float16).numpy())
w.write_header_to_file()
w.write_kv_data_to_file()
w.write_tensors_to_file()
w.close()
def _affianca(t, nk=16, nvpk=2, hd=128):
s = list(t.shape)
t = t.reshape(nk, nvpk, hd, *s[1:])
return t.permute(1, 0, 2, *range(3, t.ndim)).contiguous().reshape(s)
# ---- Shared prefix. K targets on the same screenshot = one sequence X = [prefix | suffix 1 | ... | suffix K] (lens = lengths).
# Projections, norms and MLP run on all tokens together (one weight unpack); causal conv, GDN recurrence and attention run per
# segment: the prefix alone, every branch with the prefix's state / keys. Same math as Qwen3_5DecoderLayer on each full
# sequence (different summation order: within tolerance, not bit-exact).
def _gdn(g, x, lens):
n, k = x.shape[1], g.conv_kernel_size
mixed = g.in_proj_qkv(x).transpose(1, 2) # (1, C, T)
z = g.in_proj_z(x).reshape(1, n, -1, g.head_v_dim)
beta, a = g.in_proj_b(x).sigmoid(), g.in_proj_a(x)
seg = mixed.split(lens, -1)
w, ctx = g.conv1d.weight, seg[0][..., -(k - 1):] # branches see the last k-1 REAL inputs of the prefix, not zeros
conv = lambda s: F.conv1d(s, w, None, groups=w.shape[0]) # noqa: E731
mixed = torch.cat([conv(F.pad(seg[0], (k - 1, 0)))] + [conv(torch.cat([ctx, s], -1)) for s in seg[1:]], -1)
mixed = mq.ACT2FN[g.activation](mixed).transpose(1, 2)
q, kk, v = torch.split(mixed, [g.key_dim, g.key_dim, g.value_dim], -1)
q, kk, v = (t.reshape(1, n, -1, d) for t, d in ((q, g.head_k_dim), (kk, g.head_k_dim), (v, g.head_v_dim)))
rip = g.num_v_heads // g.num_k_heads
if rip > 1:
q, kk = q.repeat_interleave(rip, 2), kk.repeat_interleave(rip, 2)
dec = -g.A_log.float().exp() * F.softplus(a.float() + g.dt_bias)
out, stato = [], None
for i, (qi, ki, vi, di, bi) in enumerate(zip(*(t.split(lens, 1) for t in (q, kk, v, dec, beta)))):
o, s = mq.torch_chunk_gated_delta_rule(qi, ki, vi, g=di, beta=bi, initial_state=stato, output_final_state=i == 0,
use_qk_l2norm_in_kernel=True)
out.append(o)
stato = s if i == 0 else stato # state after the prefix (padding to 64 uses k=v=beta=0, g=0, so it stays unchanged)
o = g.norm(torch.cat(out, 1).reshape(-1, g.head_v_dim), z.reshape(-1, g.head_v_dim)).reshape(1, n, -1)
return g.out_proj(o)
def _attn(at, x, cos, sin, lens):
n = x.shape[1]
sh = (1, n, -1, at.head_dim)
q, gate = torch.chunk(at.q_proj(x).view(1, n, -1, at.head_dim * 2), 2, -1)
q, k = mq.apply_rotary_pos_emb(at.q_norm(q.reshape(sh)).transpose(1, 2), at.k_norm(at.k_proj(x).view(sh)).transpose(1, 2), cos, sin)
v = at.v_proj(x).view(sh).transpose(1, 2)
qs, ks, vs = q.split(lens, 2), k.split(lens, 2), v.split(lens, 2)
f = mq.ALL_ATTENTION_FUNCTIONS.get_interface(at.config._attn_implementation, mq.eager_attention_forward)
out = [f(at, qs[0], ks[0], vs[0], None, dropout=0.0, scaling=at.scaling)[0]] # prefix: causal
for qi, ki, vi in zip(qs[1:], ks[1:], vs[1:]): # branch: keys = prefix + own suffix, each query sees the prefix and its past
kk, vv = torch.cat([ks[0], ki], 2), torch.cat([vs[0], vi], 2)
mask = torch.ones(qi.shape[2], kk.shape[2], dtype=torch.bool, device=x.device).tril(lens[0])[None, None]
out.append(f(at, qi, kk, vv, mask, dropout=0.0, scaling=at.scaling)[0])
return at.o_proj(torch.cat(out, 1).reshape(1, n, -1) * torch.sigmoid(gate.reshape(1, n, -1)))
def strato_ramificato(L, X, cos, sin, lens):
h = L.input_layernorm(X)
X = X + (_gdn(L.linear_attn, h, lens) if L.block_type == "linear_attention" else _attn(L.self_attn, h, cos, sin, lens))
return X + L.mlp(L.post_attention_layernorm(X))