Download package/robin/lora.py from FixedSeed/Robin: direct link, hf CLI and curl.
- Browser
- Download file 4.23 kB
-
https://huggingface.co/FixedSeed/Robin/resolve/main/package/robin/lora.py
- Command line
-
hf download hf://FixedSeed/Robin/package/robin/lora.py
-
curl -L -o lora.py https://huggingface.co/FixedSeed/Robin/resolve/main/package/robin/lora.py
4.23 kB
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import torch | |
| from safetensors import safe_open | |
| from safetensors.torch import save_file | |
| from torch import nn | |
| TARGETS = ('self_attn.qkv_proj', 'self_attn.o_proj', 'mlp.gate_up_proj') | |
| FORMAT = 'robin-lora-1' | |
| def file_sha256(path): | |
| path = Path(path).resolve(); st = path.stat() | |
| key = f'{path}:{st.st_size}:{st.st_mtime_ns}' | |
| cache = Path(os.environ.get('ROBIN_CACHE', Path.home()/'.cache'/'robin'))/'sha256.json' | |
| known = {} | |
| try: known = json.loads(cache.read_text()) | |
| except (OSError, ValueError): pass | |
| if key not in known: | |
| h = hashlib.sha256() | |
| with path.open('rb') as f: | |
| for chunk in iter(lambda: f.read(1 << 24), b''): h.update(chunk) | |
| known[key] = h.hexdigest() | |
| try: | |
| cache.parent.mkdir(parents=True, exist_ok=True); cache.write_text(json.dumps(known)) | |
| except OSError: pass | |
| return known[key] | |
| class LoRA(nn.Module): | |
| def __init__(self, nets, ranks, alphas=None, targets=TARGETS, seed=0): | |
| super().__init__() | |
| self.ranks, self.alphas, self.targets = dict(ranks), dict(alphas or ranks), tuple(targets) | |
| self.scale, self.names, self._mods, self._factor, self._handles = 1.0, [], [], [], [] | |
| self.A, self.B = nn.ParameterList(), nn.ParameterList() | |
| g = torch.Generator().manual_seed(seed) | |
| for net, layers in nets.items(): | |
| r = self.ranks[net] | |
| for i, layer in enumerate(layers): | |
| for t in self.targets: | |
| m = layer.get_submodule(t) | |
| a = torch.randn(r, m.in_features, generator=g) / math.sqrt(m.in_features) | |
| self.A.append(nn.Parameter(a.to(m.weight.device))) | |
| self.B.append(nn.Parameter(torch.zeros(m.out_features, r, device=m.weight.device))) | |
| self.names.append(f'{net}.layers.{i}.{t}'); self._mods.append(m); self._factor.append(self.alphas[net] / r) | |
| def attach(self): | |
| if not self._handles: | |
| self._handles = [m.register_forward_hook(self._hook(i)) for i, m in enumerate(self._mods)] | |
| return self | |
| def detach(self): | |
| for h in self._handles: h.remove() | |
| self._handles = [] | |
| def _hook(self, i): | |
| def hook(_module, args, out): | |
| if self.scale == 0: return out | |
| delta = (args[0].float() @ self.A[i].T) @ self.B[i].T | |
| return out + (delta * (self._factor[i] * self.scale)).to(out.dtype) | |
| return hook | |
| def config(self, base_sha256=None): | |
| return dict(format=FORMAT, ranks=self.ranks, alphas=self.alphas, targets=list(self.targets), base_sha256=base_sha256) | |
| def save(self, path, base_sha256=None, dtype=torch.bfloat16): | |
| path = Path(path); tmp = path.with_name(path.name + '.tmp') | |
| state = {} | |
| for name, a, b in zip(self.names, self.A, self.B): | |
| state[f'{name}.lora_A'] = a.detach().to(dtype).cpu().contiguous(); state[f'{name}.lora_B'] = b.detach().to(dtype).cpu().contiguous() | |
| save_file(state, str(tmp), metadata=dict(robin_lora=json.dumps(self.config(base_sha256)))) | |
| os.replace(tmp, path) | |
| def load(cls, path, nets, device='cuda'): | |
| with safe_open(str(path), framework='pt', device='cpu') as f: | |
| meta = f.metadata() or {} | |
| if 'robin_lora' not in meta: | |
| raise ValueError(f'{path} is not a Robin LoRA') | |
| config = json.loads(meta['robin_lora']) | |
| lora = cls({k: v for k, v in nets.items() if k in config['ranks']}, config['ranks'], config['alphas'], config['targets']) | |
| keys = set(f.keys()) | |
| for i, name in enumerate(lora.names): | |
| for which, params in (('lora_A', lora.A), ('lora_B', lora.B)): | |
| k = f'{name}.{which}' | |
| if k not in keys: raise ValueError(f'{path} is missing {k}') | |
| t = f.get_tensor(k) | |
| if t.shape != params[i].shape: raise ValueError(f'{path}: {k} has shape {tuple(t.shape)}, expected {tuple(params[i].shape)}') | |
| params[i].data.copy_(t) | |
| return lora.to(device).requires_grad_(False), config | |