File size: 16,113 Bytes
92f6dce
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
"""LittleBit compression of micro700kb (Model2Vec vocab-quantized static model).

Original layout: embeddings int8 [K=2000, 256] (shared vectors), mapping int32 [V] (token -> vector),
weights fp32 [V] (per-token scale). Token vector = weights[t] * embeddings[mapping[t]].

Compressed layout (.lbit):
  centroids.{pri,res}.{U_bits,V_bits,h,g,l}   LittleBit factors of the 2000 x 256 table
  mapping_bits   uint8   mapping packed at ceil(log2 K) bits per token (lossless)
  weights        fp16    per-token scale (trainable during retraining)

  python micro_lb.py --bpw 1.0 --steps 6000
  python micro_lb.py --baseline          # just re-pack mapping/weights, no LittleBit
"""
import argparse, json, math, os, random, time, torch, torch.nn as nn, torch.nn.functional as F
from safetensors import safe_open
from safetensors.torch import load_file, save_file
from tokenizers import Tokenizer

HERE = os.path.dirname(os.path.abspath(__file__))
p = argparse.ArgumentParser()
p.add_argument("--bpw", type=float, default=1.0, help="bits per weight for the 2000x256 centroid table")
p.add_argument("--steps", type=int, default=6000)
p.add_argument("--bs", type=int, default=512)
p.add_argument("--lr", type=float, default=3e-3)
p.add_argument("--no_residual", action="store_true")
p.add_argument("--freeze_weights", action="store_true", help="keep per-token scales fixed")
p.add_argument("--baseline", action="store_true")
p.add_argument("--teacher", default="self", choices=["self", "bge", "bge-small"],
               help="self = copy the original micro model; bge / bge-small = learn from BAAI/bge-{base,small}-en-v1.5")
p.add_argument("--alpha", type=float, default=1.0,
               help="teacher similarity = alpha * model + (1 - alpha) * BM25F. 1.0 = model only")
p.add_argument("--title_w", type=float, default=2.0, help="BM25F weight of the section-title field (body = 1)")
p.add_argument("--k1", type=float, default=1.2)
p.add_argument("--b", type=float, default=0.75)
p.add_argument("--idf_n", type=int, default=300_000, help="sentences used to estimate IDF / field lengths")
p.add_argument("--no_retrieval", action="store_true", help="skip the SciFact retrieval eval")
p.add_argument("--teacher_bs", type=int, default=128, help="sub-batch size for the bge forward pass")
args = p.parse_args()
dev = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)

# ---------------- original model ----------------
t = load_file(os.path.join(HERE, "orig", "model.safetensors"))
C0 = t["embeddings"].float().to(dev)          # global scale is irrelevant after L2-normalisation
MAP = t["mapping"].long().to(dev)
WT0 = t["weights"].float().to(dev)
K, D = C0.shape; V = MAP.numel()
tok = Tokenizer.from_file(os.path.join(HERE, "orig", "tokenizer.json"))
def table(C, wt): return wt[:, None] * C[MAP]
T0 = table(C0, WT0)

# ---------------- LittleBit core ----------------
class SmoothSign(torch.autograd.Function):
    @staticmethod
    def forward(ctx, x): ctx.save_for_backward(x); return torch.where(x >= 0, 1.0, -1.0).to(x.dtype)
    @staticmethod
    def backward(ctx, g): x, = ctx.saved_tensors; th = torch.tanh(100 * x); return g * 100 * (1 - th * th)

def rank1(M):
    U, S, Vh = torch.linalg.svd(M, full_matrices=False)
    a, b = U[:, 0] * S[0].sqrt(), Vh[0] * S[0].sqrt()
    if a.sum() < 0: a, b = -a, -b
    return a.abs().clamp_min(1e-8), b.abs().clamp_min(1e-8)

class Path(nn.Module):
    def __init__(self, W, r):
        super().__init__()
        U, S, Vh = torch.linalg.svd(W, full_matrices=False)
        s = S[:r].sqrt(); Up, Vp = U[:, :r] * s, Vh[:r].T * s
        h, lu = rank1(Up.abs()); g, lv = rank1(Vp.abs())
        self.U, self.V = nn.Parameter(Up), nn.Parameter(Vp)
        self.h, self.g, self.l = nn.Parameter(h), nn.Parameter(g), nn.Parameter(lu * lv)
    def weight(self):
        return (self.h[:, None] * SmoothSign.apply(self.U) * self.l) @ (SmoothSign.apply(self.V) * self.g[:, None]).T

class Student(nn.Module):
    def __init__(self, r, residual):
        super().__init__()
        self.pri = Path(C0, r)
        self.res = None
        if residual:
            with torch.no_grad(): self.res = Path(C0 - self.pri.weight(), r)
        self.wt = nn.Parameter(WT0.clone(), requires_grad=not args.freeze_weights)
    def centroids(self): return self.pri.weight() + (self.res.weight() if self.res is not None else 0)
    def table(self): return table(self.centroids(), self.wt)

def rank_for(bpw, n, d, npath): return max(1, int((bpw * n * d / npath - 16 * (n + d)) / (n + d + 16)))

# ---------------- packing ----------------
SH = torch.tensor([7, 6, 5, 4, 3, 2, 1, 0], dtype=torch.uint8)
def pack_signs(x):
    rows, r = x.shape; b = (x >= 0).to(torch.uint8)
    if r % 8: b = torch.cat([b, torch.zeros(rows, (-r) % 8, dtype=torch.uint8)], 1)
    return (b.view(rows, -1, 8) << SH).sum(-1).to(torch.uint8)
def unpack_signs(pk, r): return ((pk.unsqueeze(-1) >> SH) & 1).view(pk.shape[0], -1)[:, :r].float() * 2 - 1

def pack_ints(x, nbits):
    """little-endian bit stream of non-negative ints, nbits each -> uint8"""
    x = x.cpu().long()
    bits = ((x.unsqueeze(1) >> torch.arange(nbits)) & 1).view(-1).to(torch.uint8)
    if bits.numel() % 8: bits = torch.cat([bits, torch.zeros((-bits.numel()) % 8, dtype=torch.uint8)])
    return (bits.view(-1, 8) << torch.arange(8, dtype=torch.uint8)).sum(-1).to(torch.uint8)
def unpack_ints(b, nbits, n):
    bits = ((b.long().unsqueeze(1) >> torch.arange(8)) & 1).view(-1)[: n * nbits].view(n, nbits)
    return (bits << torch.arange(nbits)).sum(-1)

NB = max(1, math.ceil(math.log2(K)))
def save(path, student=None, r=0):
    out = {"mapping_bits": pack_ints(MAP, NB)}
    if student is None:   # baseline: original int8 centroids, re-packed mapping, fp16 weights
        out["centroids.int8"] = t["embeddings"]; out["weights"] = WT0.cpu().half()
    else:
        for nm in ("pri", "res"):
            pth = getattr(student, nm)
            if pth is None: continue
            out[f"centroids.{nm}.U_bits"] = pack_signs(pth.U.detach().cpu())
            out[f"centroids.{nm}.V_bits"] = pack_signs(pth.V.detach().cpu())
            for s in "hgl": out[f"centroids.{nm}.{s}"] = getattr(pth, s).detach().cpu().half()
        out["weights"] = student.wt.detach().cpu().half()
    save_file(out, path, metadata={"format": "littlebit-model2vec-vq-v1", "rank": str(r), "K": str(K),
                                   "vocab": str(V), "map_bits": str(NB)})

def load_table(path):
    with safe_open(path, "pt") as f: m = f.metadata()
    x = load_file(path); r, n, nb = int(m["rank"]), int(m["vocab"]), int(m["map_bits"])
    mp = unpack_ints(x["mapping_bits"], nb, n)
    if "centroids.int8" in x: C = x["centroids.int8"].float()
    else:
        C = 0
        for nm in ("pri", "res"):
            if f"centroids.{nm}.U_bits" not in x: continue
            Us, Vs = unpack_signs(x[f"centroids.{nm}.U_bits"], r), unpack_signs(x[f"centroids.{nm}.V_bits"], r)
            h, g, l = (x[f"centroids.{nm}.{s}"].float() for s in "hgl")
            C = C + (h[:, None] * Us * l) @ (Vs * g[:, None]).T
    return x["weights"].float()[:, None] * C[mp]

# ---------------- data + eval ----------------
from datasets import load_dataset
def ids_of(texts):
    flat, offs = [], []
    for e in tok.encode_batch(texts, add_special_tokens=False):
        offs.append(len(flat)); flat.extend(e.ids if e.ids else [0])
    return torch.tensor(flat, device=dev), torch.tensor(offs, device=dev)
def embed(W, flat, offs): return F.normalize(F.embedding_bag(flat, W, offs, mode="mean"), dim=-1)

sts = load_dataset("sentence-transformers/stsb", split="test")
sa, sb = ids_of(sts["sentence1"]), ids_of(sts["sentence2"])
gold = torch.tensor(sts["score"])
def sts_spearman(W):
    with torch.no_grad(): cos = (embed(W.to(dev), *sa) * embed(W.to(dev), *sb)).sum(-1).cpu()
    rk = lambda x: x.argsort().argsort().float()
    return torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0, 1].item()

def raw_pairs():
    """(section title, sentence) from streamed WikiText-103, in order. Title = latest heading."""
    title = ""
    for row in load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train", streaming=True):
        l = row["text"].strip()
        if l.startswith("=") and l.endswith("="):
            title = l.strip("= ").strip(); continue
        if len(l) <= 20: continue
        for s in l.split(". "):
            s = s.strip()
            if len(s) > 10: yield title, s

def sentence_stream(seed=0, buffer=50_000):
    rng = random.Random(seed); buf = []
    while True:
        for pair in raw_pairs():
            if len(buf) < buffer: buf.append(pair); continue
            i = rng.randrange(buffer); yield buf[i]; buf[i] = pair

# ---------------- BM25F (fields: title, body) over the model's own token ids ----------------
def bm25f_stats():
    path = os.path.join(HERE, f"bm25_stats_{args.idf_n}.pt")
    if os.path.exists(path): return torch.load(path)
    df = torch.zeros(V); n = 0; len_b = len_t = 0.0; chunk = []
    def flush(chunk):
        nonlocal n, len_b, len_t
        eb = tok.encode_batch([s for _, s in chunk], add_special_tokens=False)
        et = tok.encode_batch([t for t, _ in chunk], add_special_tokens=False)
        for b_, t_ in zip(eb, et):
            ids = set(b_.ids) | set(t_.ids)
            if ids: df[list(ids)] += 1
            len_b += len(b_.ids); len_t += len(t_.ids); n += 1
    for pair in raw_pairs():
        chunk.append(pair)
        if len(chunk) == 10_000: flush(chunk); chunk = []
        if n + len(chunk) >= args.idf_n: break
    if chunk: flush(chunk)
    st_ = dict(idf=torch.log(1 + (n - df + 0.5) / (df + 0.5)), avg_b=len_b / n, avg_t=len_t / n, n=n)
    torch.save(st_, path); return st_

def bm25f_sim(pairs, stats):
    """Symmetric, self-normalised BM25F similarity between all sentences in a batch -> [B, B] in ~[0, 1]."""
    B = len(pairs)
    eb = tok.encode_batch([s for _, s in pairs], add_special_tokens=False)
    et = tok.encode_batch([t for t, _ in pairs], add_special_tokens=False)
    def tf_matrix(encs, avg):
        rows = torch.cat([torch.full((len(e.ids),), i) for i, e in enumerate(encs)] + [torch.zeros(0, dtype=torch.long)]).long()
        cols = torch.tensor([j for e in encs for j in e.ids], dtype=torch.long)
        M = torch.zeros(B, V, device=dev)
        M.index_put_((rows.to(dev), cols.to(dev)), torch.ones(len(cols), device=dev), accumulate=True)
        lens = torch.tensor([len(e.ids) for e in encs], device=dev, dtype=torch.float)
        return M / (1 - args.b + args.b * lens[:, None] / avg)
    tf = tf_matrix(eb, stats["avg_b"]) + args.title_w * tf_matrix(et, stats["avg_t"])   # BM25F pseudo-tf
    sat = tf / (args.k1 + tf)                                                           # saturation
    q = (tf_matrix(eb, stats["avg_b"]) > 0).float() * stats["idf"].to(dev)              # query = body terms, idf-weighted
    S = q @ sat.T                                     # S[i, j] = BM25F(query i -> doc j)
    S = (S + S.T) / 2
    d = S.diag().clamp_min(1e-6).sqrt()
    return (S / d[:, None] / d[None, :]).clamp(0, 1)

# ---------------- retrieval eval: SciFact (BEIR), nDCG@10 ----------------
def load_scifact():
    corpus = load_dataset("mteb/scifact", "corpus", split="corpus")
    queries = load_dataset("mteb/scifact", "queries", split="queries")
    qrels = load_dataset("mteb/scifact", "default", split="test")
    qrel = {}
    for r in qrels:
        if r["score"] > 0: qrel.setdefault(str(r["query-id"]), set()).add(str(r["corpus-id"]))
    qs = [(str(q["_id"]), q["text"]) for q in queries if str(q["_id"]) in qrel]
    docs = [(str(c["_id"]), (c.get("title") or "") + " " + c["text"]) for c in corpus]
    return docs, qs, qrel
SCIFACT = None if args.no_retrieval else load_scifact()

def ndcg10(W):
    docs, qs, qrel = SCIFACT
    with torch.no_grad():
        De = embed(W.to(dev), *ids_of([d for _, d in docs])); Qe = embed(W.to(dev), *ids_of([q for _, q in qs]))
        top = (Qe @ De.T).topk(10, dim=1).indices.cpu()
    tot = 0.0
    for qi, (qid, _) in enumerate(qs):
        rel = qrel[qid]
        dcg = sum(1 / math.log2(r + 2) for r, di in enumerate(top[qi].tolist()) if docs[di][0] in rel)
        idcg = sum(1 / math.log2(r + 2) for r in range(min(len(rel), 10)))
        tot += dcg / idcg
    return tot / len(qs)
def report(name, W):
    msg = f"{name:10s} STS-B {sts_spearman(W):.4f}"
    if SCIFACT: msg += f" | SciFact nDCG@10 {ndcg10(W):.4f}"
    print(msg, flush=True)

orig_kb = os.path.getsize(os.path.join(HERE, "orig", "model.safetensors")) / 1e3
print(f"original micro700kb: {orig_kb:.0f} KB | K={K} centroids x {D}, vocab {V}", flush=True)
print(f"original STS-B spearman {sts_spearman(T0):.4f}", flush=True)

if args.baseline:
    save("micro_repacked.lbit")
    print(f"re-packed (int8 centroids, {NB}-bit mapping, fp16 weights): "
          f"{os.path.getsize('micro_repacked.lbit')/1e3:.0f} KB, STS-B {sts_spearman(load_table('micro_repacked.lbit')):.4f}")
    raise SystemExit

npath = 1 if args.no_residual else 2
r = rank_for(args.bpw, K, D, npath)
st = Student(r, not args.no_residual).to(dev)
with torch.no_grad(): print(f"LittleBit {args.bpw} BPW centroids: rank {r}, {npath} path(s) | init STS-B {sts_spearman(st.table()):.4f}", flush=True)

bge = None
if args.teacher in ("bge", "bge-small"):
    from sentence_transformers import SentenceTransformer
    bge = SentenceTransformer("BAAI/bge-base-en-v1.5" if args.teacher == "bge" else "BAAI/bge-small-en-v1.5", device=dev)
    bge.half()
    with torch.no_grad():
        a = bge.encode(list(sts["sentence1"]), batch_size=128, convert_to_tensor=True, normalize_embeddings=True)
        b = bge.encode(list(sts["sentence2"]), batch_size=128, convert_to_tensor=True, normalize_embeddings=True)
    cos = (a.float() * b.float()).sum(-1).cpu(); rk = lambda x: x.argsort().argsort().float()
    print(f"teacher {args.teacher} STS-B spearman {torch.corrcoef(torch.stack([rk(cos), rk(gold)]))[0,1].item():.4f} (ceiling)", flush=True)

def teacher_emb(texts, flat, offs):
    if bge is None:
        return embed(T0, flat, offs)
    return bge.encode(texts, batch_size=args.teacher_bs, convert_to_tensor=True,
                      normalize_embeddings=True, show_progress_bar=False).float()

BM25 = bm25f_stats() if args.alpha < 1 else None
if BM25: print(f"BM25F stats from {BM25['n']:,} sentences (avg body {BM25['avg_b']:.1f}, title {BM25['avg_t']:.1f} tokens)", flush=True)
report("original", T0)
opt = torch.optim.Adam([q for q in st.parameters() if q.requires_grad], lr=args.lr)
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=args.lr, total_steps=max(4, args.steps),
                                            pct_start=max(0.02, 2 / max(4, args.steps)))  # short runs need >=2 warmup steps
stream = sentence_stream(); t0 = time.time()
for step in range(args.steps):
    pairs = [next(stream) for _ in range(args.bs)]
    texts = [s_ for _, s_ in pairs]
    flat, offs = ids_of(texts)
    with torch.no_grad():
        tgt = teacher_emb(texts, flat, offs); T = tgt @ tgt.T
        if args.alpha < 1: T = args.alpha * T + (1 - args.alpha) * bm25f_sim(pairs, BM25)
    s = embed(st.table(), flat, offs)
    loss = F.mse_loss(s @ s.T, T) * 100       # relational distillation (what worked for potion)
    opt.zero_grad(); loss.backward(); opt.step(); sched.step()
    if step % 1000 == 0 or step == args.steps - 1:
        print(f"step {step} loss {loss.item():.4f} {time.time()-t0:.0f}s", flush=True)

tag = f"{args.bpw}{'_nores' if args.no_residual else ''}{'_' + args.teacher if args.teacher != 'self' else ''}{f'_a{args.alpha}' if args.alpha < 1 else ''}"
out = f"micro_lb_{tag}.lbit"; save(out, st, r)
res = dict(bpw=args.bpw, rank=r, steps=args.steps, orig_kb=orig_kb, file_kb=os.path.getsize(out) / 1e3,
           orig_sts=sts_spearman(T0), final_sts=sts_spearman(load_table(out)))
print(f"wrote {out}: {res['file_kb']:.0f} KB (from {orig_kb:.0f} KB) | packed STS-B {res['final_sts']:.4f}", flush=True)
report("final", load_table(out))
json.dump(res, open(f"result_{tag}.json", "w"), indent=1)