File size: 3,730 Bytes
5c251fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | """Encodes every screenshot once with Bonsai's vision tower (weights from the mmproj GGUF, the same tower llama.cpp runs)
and saves the embeddings (image tokens x 4096, fp16), so the tower is not on the GPU during training.
Usage: python vision_cache.py <train.jsonl> [max_image_tokens=1024]
Paths: MMPROJ (Ternary-Bonsai-2-9B-mmproj-BF16.gguf), QWEN_SRC (config.json + preprocessor_config.json from
Qwen/Qwen3.5-9B), CACHE_DIR (output folder), GGUF_PY (gguf-py of the PrismML fork)."""
import json
import os
import sys
import numpy as np
import torch
from PIL import Image
from transformers import AutoImageProcessor
from transformers.models.qwen3_5 import modeling_qwen3_5 as mq
sys.path.insert(0, os.environ.get("GGUF_PY", "llama.cpp/gguf-py"))
import gguf # noqa: E402
MMPROJ = os.environ.get("MMPROJ", "Ternary-Bonsai-2-9B-mmproj-BF16.gguf")
SRC = os.environ.get("QWEN_SRC", "qwen35-9b")
CACHE = os.environ.get("CACHE_DIR", "vision-cache")
def torre(device="cuda"):
cfg = mq.Qwen3_5Config(**json.load(open(SRC + "/config.json"))).vision_config
cfg._attn_implementation = "sdpa"
v = mq.Qwen3_5VisionModel(cfg)
r = gguf.GGUFReader(MMPROJ)
def t(nome):
x = r.get_tensor([i for i, tt in enumerate(r.tensors) if tt.name == nome][0])
a = np.array(x.data)
if x.tensor_type == gguf.GGMLQuantizationType.BF16:
a = (a.view(np.uint16).astype(np.uint32) << 16).view(np.float32)
return torch.from_numpy(a.astype(np.float32).reshape([int(s) for s in reversed(x.shape)]))
sd = {"patch_embed.proj.weight": torch.stack([t("v.patch_embd.weight"), t("v.patch_embd.weight.1")], 2),
"patch_embed.proj.bias": t("v.patch_embd.bias"), "pos_embed.weight": t("v.position_embd.weight"),
"merger.norm.weight": t("v.post_ln.weight"), "merger.norm.bias": t("v.post_ln.bias"),
"merger.linear_fc1.weight": t("mm.0.weight"), "merger.linear_fc1.bias": t("mm.0.bias"),
"merger.linear_fc2.weight": t("mm.2.weight"), "merger.linear_fc2.bias": t("mm.2.bias")}
for i in range(cfg.depth):
for hf, gg in (("attn.qkv", "attn_qkv"), ("attn.proj", "attn_out"), ("mlp.linear_fc1", "ffn_up"), ("mlp.linear_fc2", "ffn_down"),
("norm1", "ln1"), ("norm2", "ln2")):
for k in ("weight", "bias"):
sd[f"blocks.{i}.{hf}.{k}"] = t(f"v.blk.{i}.{gg}.{k}")
mancanti = v.load_state_dict(sd, strict=False)
assert not mancanti.unexpected_keys, mancanti
assert all("rotary" in k for k in mancanti.missing_keys), mancanti.missing_keys
return v.to(device).float().eval()
def proc(max_tok):
p = AutoImageProcessor.from_pretrained(SRC)
p.max_pixels = max_tok * 32 * 32
p.min_pixels = 64 * 32 * 32
p.size = {"shortest_edge": p.min_pixels, "longest_edge": p.max_pixels}
return p
if __name__ == "__main__":
righe = [json.loads(line) for line in open(sys.argv[1], encoding="utf-8")]
mt = int(sys.argv[2]) if len(sys.argv) > 2 else 1024
os.makedirs(CACHE, exist_ok=True)
v, p = torre(), proc(mt)
fatti = 0
for r in {r["cache"]: r for r in righe}.values():
f = f"{CACHE}/{r['cache']}.pt"
if os.path.exists(f):
continue
x = p(images=[Image.open(r["img"]).convert("RGB")], return_tensors="pt")
with torch.no_grad():
e = v(x["pixel_values"].cuda().float(), grid_thw=x["image_grid_thw"].cuda())
e = e.pooler_output if hasattr(e, "pooler_output") else e
torch.save({"emb": e.half().cpu(), "thw": x["image_grid_thw"][0]}, f + ".tmp")
os.replace(f + ".tmp", f)
fatti += 1
if fatti % 50 == 0:
print("encoded", fatti, flush=True)
print("done", fatti)
|