"""Decode a micro700kb-style model to a dense [vocab, 256] numpy table. Accepts either the original Model2Vec safetensors (embeddings/mapping/weights) or a LittleBit .lbit written by micro_lb.py.""" import torch from safetensors import safe_open from safetensors.torch import load_file _SH = torch.tensor([7, 6, 5, 4, 3, 2, 1, 0], dtype=torch.uint8) def _signs(pk, r): return ((pk.unsqueeze(-1) >> _SH) & 1).view(pk.shape[0], -1)[:, :r].float() * 2 - 1 def _ints(b, nbits, n): bits = ((b.long().unsqueeze(1) >> torch.arange(8)) & 1).view(-1)[: n * nbits].view(n, nbits) return (bits << torch.arange(nbits)).sum(-1) def decode(path): x = load_file(path) if "embeddings" in x: # original vocab-quantized Model2Vec return (x["weights"].float()[:, None] * x["embeddings"].float()[x["mapping"].long()]).numpy() with safe_open(path, "pt") as f: m = f.metadata() r, n, nb = int(m["rank"]), int(m["vocab"]), int(m["map_bits"]) C = 0 for p in ("pri", "res"): if f"centroids.{p}.U_bits" not in x: continue h, g, l = (x[f"centroids.{p}.{s}"].float() for s in "hgl") C = C + (h[:, None] * _signs(x[f"centroids.{p}.U_bits"], r) * l) @ (_signs(x[f"centroids.{p}.V_bits"], r) * g[:, None]).T return (x["weights"].float()[:, None] * C[_ints(x["mapping_bits"], nb, n)]).numpy()