File size: 4,230 Bytes
5e88ff2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
import hashlib
import json
import math
import os
from pathlib import Path

import torch
from safetensors import safe_open
from safetensors.torch import save_file
from torch import nn

TARGETS = ('self_attn.qkv_proj', 'self_attn.o_proj', 'mlp.gate_up_proj')
FORMAT = 'robin-lora-1'


def file_sha256(path):
    path = Path(path).resolve(); st = path.stat()
    key = f'{path}:{st.st_size}:{st.st_mtime_ns}'
    cache = Path(os.environ.get('ROBIN_CACHE', Path.home()/'.cache'/'robin'))/'sha256.json'
    known = {}
    try: known = json.loads(cache.read_text())
    except (OSError, ValueError): pass
    if key not in known:
        h = hashlib.sha256()
        with path.open('rb') as f:
            for chunk in iter(lambda: f.read(1 << 24), b''): h.update(chunk)
        known[key] = h.hexdigest()
        try:
            cache.parent.mkdir(parents=True, exist_ok=True); cache.write_text(json.dumps(known))
        except OSError: pass
    return known[key]


class LoRA(nn.Module):
    def __init__(self, nets, ranks, alphas=None, targets=TARGETS, seed=0):
        super().__init__()
        self.ranks, self.alphas, self.targets = dict(ranks), dict(alphas or ranks), tuple(targets)
        self.scale, self.names, self._mods, self._factor, self._handles = 1.0, [], [], [], []
        self.A, self.B = nn.ParameterList(), nn.ParameterList()
        g = torch.Generator().manual_seed(seed)
        for net, layers in nets.items():
            r = self.ranks[net]
            for i, layer in enumerate(layers):
                for t in self.targets:
                    m = layer.get_submodule(t)
                    a = torch.randn(r, m.in_features, generator=g) / math.sqrt(m.in_features)
                    self.A.append(nn.Parameter(a.to(m.weight.device)))
                    self.B.append(nn.Parameter(torch.zeros(m.out_features, r, device=m.weight.device)))
                    self.names.append(f'{net}.layers.{i}.{t}'); self._mods.append(m); self._factor.append(self.alphas[net] / r)

    def attach(self):
        if not self._handles:
            self._handles = [m.register_forward_hook(self._hook(i)) for i, m in enumerate(self._mods)]
        return self

    def detach(self):
        for h in self._handles: h.remove()
        self._handles = []

    def _hook(self, i):
        def hook(_module, args, out):
            if self.scale == 0: return out
            delta = (args[0].float() @ self.A[i].T) @ self.B[i].T
            return out + (delta * (self._factor[i] * self.scale)).to(out.dtype)
        return hook

    def config(self, base_sha256=None):
        return dict(format=FORMAT, ranks=self.ranks, alphas=self.alphas, targets=list(self.targets), base_sha256=base_sha256)

    def save(self, path, base_sha256=None, dtype=torch.bfloat16):
        path = Path(path); tmp = path.with_name(path.name + '.tmp')
        state = {}
        for name, a, b in zip(self.names, self.A, self.B):
            state[f'{name}.lora_A'] = a.detach().to(dtype).cpu().contiguous(); state[f'{name}.lora_B'] = b.detach().to(dtype).cpu().contiguous()
        save_file(state, str(tmp), metadata=dict(robin_lora=json.dumps(self.config(base_sha256))))
        os.replace(tmp, path)

    @classmethod
    def load(cls, path, nets, device='cuda'):
        with safe_open(str(path), framework='pt', device='cpu') as f:
            meta = f.metadata() or {}
            if 'robin_lora' not in meta:
                raise ValueError(f'{path} is not a Robin LoRA')
            config = json.loads(meta['robin_lora'])
            lora = cls({k: v for k, v in nets.items() if k in config['ranks']}, config['ranks'], config['alphas'], config['targets'])
            keys = set(f.keys())
            for i, name in enumerate(lora.names):
                for which, params in (('lora_A', lora.A), ('lora_B', lora.B)):
                    k = f'{name}.{which}'
                    if k not in keys: raise ValueError(f'{path} is missing {k}')
                    t = f.get_tensor(k)
                    if t.shape != params[i].shape: raise ValueError(f'{path}: {k} has shape {tuple(t.shape)}, expected {tuple(params[i].shape)}')
                    params[i].data.copy_(t)
        return lora.to(device).requires_grad_(False), config