yasserrmd's picture
WSP 2.0 sparse weight edit: strict-JSON fence repair (32,768 BF16 values, 16 w2 columns)
1a2bb00 verified
Raw History Blame Contribute Delete
3.99 kB
"""WSP 2.0 sparse reversible weight patches. No gradients. FP32 compute on copies, cast to stored dtype, count bitwise changes."""
import json, hashlib, torch
from safetensors.torch import save_file, load_file
torch.set_grad_enabled(False)
def _bits(t):
return t.contiguous().view(torch.int16) if t.dtype in (torch.bfloat16, torch.float16) else t.contiguous().view(torch.int32)
def tensor_sha(t):
return hashlib.sha256(t.detach().contiguous().view(torch.uint8).cpu().numpy().tobytes()).hexdigest()
def build(model, edits, dirs):
"""edits: list of {param, cols, kind: proj|scale, alpha|scale, dir: key in dirs}. Returns patch dict."""
P = dict(model.named_parameters()); out = {}; meta = []
for e in edits:
W = P[e["param"]].data; W32 = out[e["param"]]["W32"] if e["param"] in out else W.float().clone()
J = torch.as_tensor(e["cols"], device=W.device, dtype=torch.long)
old = W32[:, J].clone()
if e["kind"] == "proj":
d = dirs[e["dir"]].to(W.device).float(); d = d / d.norm()
W32[:, J] = old - e["alpha"] * torch.outer(d, d @ old)
elif e["kind"] == "scale":
W32[:, J] = old * e["scale"]
else: raise ValueError(e["kind"])
meta.append(dict(param=e["param"], kind=e["kind"], n_cols=len(e["cols"]), alpha=e.get("alpha"), scale=e.get("scale"), dir=e.get("dir"),
fp32_delta_fro=float((W32[:, J] - old).norm())))
out.setdefault(e["param"], {})["W32"] = W32
patch = {"tensors": {}, "meta": meta}
total = 0
for name, d in out.items():
W = P[name].data; new = d["W32"].to(W.dtype)
ch = (_bits(new) != _bits(W)).reshape(-1).nonzero().squeeze(-1)
patch["tensors"][name] = {"idx": ch.cpu(), "orig": W.reshape(-1)[ch].cpu().clone(), "new": new.reshape(-1)[ch].cpu().clone(),
"shape": list(W.shape), "dtype": str(W.dtype),
"stored_delta_fro": float((new.float() - W.float()).norm()), "orig_param_sha256": tensor_sha(W)}
total += ch.numel()
patch["changed_scalars"] = total
return patch
def apply(model, patch, reverse=False):
P = dict(model.named_parameters())
for name, t in patch["tensors"].items():
flat = P[name].data.view(-1); idx = t["idx"].to(flat.device)
flat[idx] = (t["orig"] if reverse else t["new"]).to(flat.device)
def verify(model, patch, state):
"""state: 'orig' or 'new'. Bitwise check of patched coordinates."""
P = dict(model.named_parameters())
for name, t in patch["tensors"].items():
flat = P[name].data.view(-1); idx = t["idx"].to(flat.device)
ref = (t["new"] if state == "new" else t["orig"]).to(flat.device)
if not torch.equal(_bits(flat[idx]), _bits(ref)): return False
return True
def save(patch, path_st, path_json, extra=None):
tens = {}
for name, t in patch["tensors"].items():
tens[f"{name}::idx"] = t["idx"].contiguous(); tens[f"{name}::orig"] = t["orig"].contiguous(); tens[f"{name}::new"] = t["new"].contiguous()
md = {"format": "wsp2-sparse-patch-v1", "changed_scalars": str(patch["changed_scalars"])}
save_file(tens, path_st, metadata=md)
j = {"meta": patch["meta"], "changed_scalars": patch["changed_scalars"],
"params": {n: {k: v for k, v in t.items() if k not in ("idx", "orig", "new")} | {"n_changed": int(t["idx"].numel())} for n, t in patch["tensors"].items()}}
if extra: j.update(extra)
j["safetensors_sha256"] = hashlib.sha256(open(path_st, "rb").read()).hexdigest()
json.dump(j, open(path_json, "w"), indent=1)
return j
def load(path_st, path_json):
tens = load_file(path_st); j = json.load(open(path_json)); patch = {"tensors": {}, "meta": j["meta"], "changed_scalars": j["changed_scalars"]}
for n, info in j["params"].items():
patch["tensors"][n] = {"idx": tens[f"{n}::idx"], "orig": tens[f"{n}::orig"], "new": tens[f"{n}::new"], **info}
return patch