Robin / package /robin /lora.py
Mothersuperior's picture
Claude Opus 5.5
Robin v1
5e88ff2
Raw History Blame Contribute Delete
4.23 kB
import hashlib
import json
import math
import os
from pathlib import Path
import torch
from safetensors import safe_open
from safetensors.torch import save_file
from torch import nn
TARGETS = ('self_attn.qkv_proj', 'self_attn.o_proj', 'mlp.gate_up_proj')
FORMAT = 'robin-lora-1'
def file_sha256(path):
path = Path(path).resolve(); st = path.stat()
key = f'{path}:{st.st_size}:{st.st_mtime_ns}'
cache = Path(os.environ.get('ROBIN_CACHE', Path.home()/'.cache'/'robin'))/'sha256.json'
known = {}
try: known = json.loads(cache.read_text())
except (OSError, ValueError): pass
if key not in known:
h = hashlib.sha256()
with path.open('rb') as f:
for chunk in iter(lambda: f.read(1 << 24), b''): h.update(chunk)
known[key] = h.hexdigest()
try:
cache.parent.mkdir(parents=True, exist_ok=True); cache.write_text(json.dumps(known))
except OSError: pass
return known[key]
class LoRA(nn.Module):
def __init__(self, nets, ranks, alphas=None, targets=TARGETS, seed=0):
super().__init__()
self.ranks, self.alphas, self.targets = dict(ranks), dict(alphas or ranks), tuple(targets)
self.scale, self.names, self._mods, self._factor, self._handles = 1.0, [], [], [], []
self.A, self.B = nn.ParameterList(), nn.ParameterList()
g = torch.Generator().manual_seed(seed)
for net, layers in nets.items():
r = self.ranks[net]
for i, layer in enumerate(layers):
for t in self.targets:
m = layer.get_submodule(t)
a = torch.randn(r, m.in_features, generator=g) / math.sqrt(m.in_features)
self.A.append(nn.Parameter(a.to(m.weight.device)))
self.B.append(nn.Parameter(torch.zeros(m.out_features, r, device=m.weight.device)))
self.names.append(f'{net}.layers.{i}.{t}'); self._mods.append(m); self._factor.append(self.alphas[net] / r)
def attach(self):
if not self._handles:
self._handles = [m.register_forward_hook(self._hook(i)) for i, m in enumerate(self._mods)]
return self
def detach(self):
for h in self._handles: h.remove()
self._handles = []
def _hook(self, i):
def hook(_module, args, out):
if self.scale == 0: return out
delta = (args[0].float() @ self.A[i].T) @ self.B[i].T
return out + (delta * (self._factor[i] * self.scale)).to(out.dtype)
return hook
def config(self, base_sha256=None):
return dict(format=FORMAT, ranks=self.ranks, alphas=self.alphas, targets=list(self.targets), base_sha256=base_sha256)
def save(self, path, base_sha256=None, dtype=torch.bfloat16):
path = Path(path); tmp = path.with_name(path.name + '.tmp')
state = {}
for name, a, b in zip(self.names, self.A, self.B):
state[f'{name}.lora_A'] = a.detach().to(dtype).cpu().contiguous(); state[f'{name}.lora_B'] = b.detach().to(dtype).cpu().contiguous()
save_file(state, str(tmp), metadata=dict(robin_lora=json.dumps(self.config(base_sha256))))
os.replace(tmp, path)
@classmethod
def load(cls, path, nets, device='cuda'):
with safe_open(str(path), framework='pt', device='cpu') as f:
meta = f.metadata() or {}
if 'robin_lora' not in meta:
raise ValueError(f'{path} is not a Robin LoRA')
config = json.loads(meta['robin_lora'])
lora = cls({k: v for k, v in nets.items() if k in config['ranks']}, config['ranks'], config['alphas'], config['targets'])
keys = set(f.keys())
for i, name in enumerate(lora.names):
for which, params in (('lora_A', lora.A), ('lora_B', lora.B)):
k = f'{name}.{which}'
if k not in keys: raise ValueError(f'{path} is missing {k}')
t = f.get_tensor(k)
if t.shape != params[i].shape: raise ValueError(f'{path}: {k} has shape {tuple(t.shape)}, expected {tuple(params[i].shape)}')
params[i].data.copy_(t)
return lora.to(device).requires_grad_(False), config