File size: 9,776 Bytes
dd973c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
"""LittleBit compression of Model2Vec potion-base-8M (static embeddings, 29528 x 256).

  python potion_lb.py --bpw 0.55            # train, eval on STS-B, write potion_0.55.lbit
  python potion_lb.py --bpw 0.55 --steps 0  # just Dual-SVID init, no retraining
  python potion_lb.py --expand potion_0.55.lbit out_dir   # write a normal Model2Vec folder

Sentence embedding (as Model2Vec): mean of token rows (no special tokens), L2-normalised.
Retraining = distillation: the original table is the teacher; loss = 1 - cos(student, teacher)
on sentence embeddings, plus a small row-level term so rare tokens are not ignored.
"""
import argparse, json, math, os, time, shutil, torch, torch.nn as nn, torch.nn.functional as F
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file, save_file, safe_open
from tokenizers import Tokenizer

REPO = "minishlab/potion-base-8M"
p = argparse.ArgumentParser()
p.add_argument("--bpw", type=float, default=0.55)
p.add_argument("--steps", type=int, default=3000)
p.add_argument("--bs", type=int, default=512)
p.add_argument("--lr", type=float, default=3e-3)
p.add_argument("--row_w", type=float, default=0.1, help="weight of the per-row MSE term")
p.add_argument("--no_residual", action="store_true")
p.add_argument("--loss", default="rel", choices=["rel", "cos"], help="rel = match pairwise similarities, cos = match each vector")
p.add_argument("--expand", nargs=2, metavar=("LBIT", "OUTDIR"))
args = p.parse_args()
dev = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)

# ---------------- LittleBit core (same maths as littlebit.py) ----------------
class SmoothSign(torch.autograd.Function):
    @staticmethod
    def forward(ctx, x):
        ctx.save_for_backward(x); return torch.where(x >= 0, 1.0, -1.0).to(x.dtype)
    @staticmethod
    def backward(ctx, g):
        x, = ctx.saved_tensors; t = torch.tanh(100 * x); return g * 100 * (1 - t * t)

def rank1(M):
    U, S, Vh = torch.linalg.svd(M, full_matrices=False)
    a, b = U[:, 0] * S[0].sqrt(), Vh[0] * S[0].sqrt()
    if a.sum() < 0: a, b = -a, -b
    return a.abs().clamp_min(1e-8), b.abs().clamp_min(1e-8)

class Path(nn.Module):
    def __init__(self, W, r):
        super().__init__()
        U, S, Vh = torch.linalg.svd(W, full_matrices=False)
        s = S[:r].sqrt(); Up, Vp = U[:, :r] * s, Vh[:r].T * s
        h, lu = rank1(Up.abs()); g, lv = rank1(Vp.abs())
        self.U, self.V = nn.Parameter(Up), nn.Parameter(Vp)
        self.h, self.g, self.l = nn.Parameter(h), nn.Parameter(g), nn.Parameter(lu * lv)
    def weight(self):
        return (self.h[:, None] * SmoothSign.apply(self.U) * self.l) @ (SmoothSign.apply(self.V) * self.g[:, None]).T

class LBTable(nn.Module):
    def __init__(self, W, r, residual):
        super().__init__()
        self.pri = Path(W, r)
        self.res = None
        if residual:
            with torch.no_grad(): self.res = Path(W - self.pri.weight(), r)
    def weight(self):
        return self.pri.weight() + (self.res.weight() if self.res is not None else 0)

def rank_for(bpw, n, d, npath):
    return max(1, int((bpw * n * d / npath - 16 * (n + d)) / (n + d + 16)))

# ---------------- packing ----------------
SH = torch.tensor([7, 6, 5, 4, 3, 2, 1, 0], dtype=torch.uint8)
def pack(x):
    rows, r = x.shape; b = (x >= 0).to(torch.uint8)
    if r % 8: b = torch.cat([b, torch.zeros(rows, (-r) % 8, dtype=torch.uint8)], 1)
    return (b.view(rows, -1, 8) << SH).sum(-1).to(torch.uint8)
def unpack(pk, r):
    return (((pk.unsqueeze(-1) >> SH) & 1).view(pk.shape[0], -1)[:, :r].float() * 2 - 1)

def save_lbit(table, path, r):
    out = {}
    for nm in ("pri", "res"):
        pth = getattr(table, nm)
        if pth is None: continue
        out[f"embeddings.{nm}.U_bits"] = pack(pth.U.detach().cpu()); out[f"embeddings.{nm}.V_bits"] = pack(pth.V.detach().cpu())
        for s in "hgl": out[f"embeddings.{nm}.{s}"] = getattr(pth, s).detach().cpu().half()
    save_file(out, path, metadata={"format": "littlebit-model2vec-v1", "base": REPO, "rank": str(r)})

def load_lbit(path):
    with safe_open(path, "pt") as f: r = int(f.metadata()["rank"])
    t = load_file(path); W = 0
    for nm in ("pri", "res"):
        if f"embeddings.{nm}.U_bits" not in t: continue
        Us, Vs = unpack(t[f"embeddings.{nm}.U_bits"], r), unpack(t[f"embeddings.{nm}.V_bits"], r)
        h, g, l = (t[f"embeddings.{nm}.{s}"].float() for s in "hgl")
        W = W + (h[:, None] * Us * l) @ (Vs * g[:, None]).T
    return W

if args.expand:
    src, outdir = args.expand
    os.makedirs(outdir, exist_ok=True)
    save_file({"embeddings": load_lbit(src).contiguous()}, os.path.join(outdir, "model.safetensors"))
    for f in ("config.json", "tokenizer.json", "modules.json", "special_tokens_map.json", "tokenizer_config.json"):
        shutil.copy(hf_hub_download(REPO, f), outdir)
    print(f"expanded {src} -> {outdir} (loadable with model2vec StaticModel.from_pretrained)"); raise SystemExit

# ---------------- model + data ----------------
E = load_file(hf_hub_download(REPO, "model.safetensors"))["embeddings"].float().to(dev)
tok = Tokenizer.from_file(hf_hub_download(REPO, "tokenizer.json"))
N, D = E.shape

def ids_of(texts):
    encs = tok.encode_batch(texts, add_special_tokens=False)
    flat, offs = [], []
    for e in encs:
        offs.append(len(flat)); flat.extend(e.ids if e.ids else [0])
    return torch.tensor(flat, device=dev), torch.tensor(offs, device=dev)

def embed(W, flat, offs):
    return F.normalize(F.embedding_bag(flat, W, offs, mode="mean"), dim=-1)

from datasets import load_dataset
import random
def sentence_stream(seed=0, buffer=50_000):
    """Streams WikiText-103 (no full download), splits paragraphs into sentence-ish chunks,
    and shuffles through a buffer. Restarts from the top if it runs out."""
    rng = random.Random(seed); buf = []
    while True:
        ds = load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train", streaming=True)
        for row in ds:
            l = row["text"].strip()
            if len(l) <= 20 or l.startswith("="): continue
            for s in l.split(". "):
                s = s.strip()
                if len(s) <= 10: continue
                if len(buf) < buffer: buf.append(s); continue
                i = rng.randrange(buffer); yield buf[i]; buf[i] = s
stream = sentence_stream()
# held-out sentences come from the validation split, never seen in training
held = [s.strip() for l in load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="validation")["text"]
        for s in l.split(". ") if len(s.strip()) > 10 and not l.strip().startswith("=")][:5000]

sts = load_dataset("sentence-transformers/stsb", split="test")
def sts_spearman(W):
    a, b = ids_of(sts["sentence1"]), ids_of(sts["sentence2"])
    with torch.no_grad():
        cos = (embed(W, *a) * embed(W, *b)).sum(-1).cpu()
    gold = torch.tensor(sts["score"])
    rk = lambda x: x.argsort().argsort().float()
    return torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0, 1].item()

# held-out agreement: mean cosine between student and teacher sentence embeddings
held_ids = ids_of(held)
def agreement(W):
    with torch.no_grad(): return (embed(W, *held_ids) * embed(E, *held_ids)).sum(-1).mean().item()

# ---------------- run ----------------
npath = 1 if args.no_residual else 2
r = rank_for(args.bpw, N, D, npath)
bits = npath * (r * (N + D) + 16 * (N + D + r))
print(f"original: {N}x{D}, fp32 {N*D*4/1e6:.1f} MB / fp16 {N*D*2/1e6:.1f} MB", flush=True)
print(f"LittleBit {args.bpw} BPW: rank {r}, {npath} path(s), {bits/8e6:.2f} MB ({bits/(N*D):.3f} BPW)", flush=True)
print(f"teacher   STS-B spearman {sts_spearman(E):.4f}", flush=True)

table = LBTable(E, r, not args.no_residual).to(dev)
with torch.no_grad(): W0 = table.weight()
print(f"init      STS-B spearman {sts_spearman(W0):.4f} | agreement {agreement(W0):.4f}", flush=True)

opt = torch.optim.Adam(table.parameters(), lr=args.lr)
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=args.lr, total_steps=max(1, args.steps), pct_start=0.02)
t0 = time.time()
for step in range(args.steps):
    batch = [next(stream) for _ in range(args.bs)]
    flat, offs = ids_of(batch)
    W = table.weight()
    with torch.no_grad(): tgt = embed(E, flat, offs)
    s = embed(W, flat, offs)
    if args.loss == "cos":
        loss = 1 - (s * tgt).sum(-1).mean()
    else:  # relational: match the batch's sentence-to-sentence similarity matrix
        loss = F.mse_loss(s @ s.T, tgt @ tgt.T) * 100
    if args.row_w:  # keep every row (incl. rare tokens) near the teacher, scaled per row
        rows = torch.randint(0, N, (4096,), device=dev)
        loss = loss + args.row_w * (1 - F.cosine_similarity(W[rows], E[rows], dim=-1).mean())
    opt.zero_grad(); loss.backward(); opt.step(); sched.step()
    if step % 500 == 0 or step == args.steps - 1:
        print(f"step {step} loss {loss.item():.4f} {time.time()-t0:.0f}s", flush=True)

with torch.no_grad(): Wf = table.weight()
res = dict(bpw=args.bpw, rank=r, mb=bits / 8e6, residual=not args.no_residual, steps=args.steps,
           teacher_sts=sts_spearman(E), init_sts=sts_spearman(W0), final_sts=sts_spearman(Wf),
           agreement=agreement(Wf))
print(f"final     STS-B spearman {res['final_sts']:.4f} | agreement {res['agreement']:.4f}", flush=True)
out = f"potion_{args.bpw}{'_nores' if args.no_residual else ''}.lbit"
save_lbit(table, out, r)
Wl = load_lbit(out).to(dev)  # round-trip check through the packed file
res["file_mb"] = os.path.getsize(out) / 1e6
res["packed_sts"] = sts_spearman(Wl)
print(f"wrote {out}: {res['file_mb']:.2f} MB, packed STS-B {res['packed_sts']:.4f}", flush=True)
json.dump(res, open(f"result_{args.bpw}{'_nores' if args.no_residual else ''}.json", "w"), indent=1)