File size: 12,083 Bytes
d59d6cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
#!/usr/bin/env python3
"""
Build tiny slices of a Qwen3.5 base model and of a LoRA decision adapter trained on it, for testing. Output is gibberish, only the shapes and code paths are real.

Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests (only the needed rows) and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. The stock script only drops layers, which cannot go below the 248320 x hidden token embedding, so this one also narrows the hidden sizes.

The adapter is narrowed with the same indices as the base: LoRA A keeps the input columns of its base weight, LoRA B its output rows (the rank is kept). Only the LoRA tensors of the kept layers are kept.

Widths follow the tinyopenjev Qwen3.5 slice: 4 layers (3 linear attention + 1 full attention), hidden 32, FFN 128, 2 heads / 1 KV head (head_dim 256 kept), linear attention 2 key heads / 6 value heads (head dims 128 kept), the MTP layer is kept (narrowed), vision encoder 2 blocks, 2 heads (head_dim kept), out hidden 32 (the text GGUF budget is 10 MB: a 248320 x 32 matrix is 4.5 MB in Q4_0).

usage:
  slice_qwen35_lora.py base <base repo> <revision|main> <output dir>
  slice_qwen35_lora.py adapter <adapter repo or local dir> <base repo> <output dir>
      (only writes the narrowed adapter_model.safetensors and the narrowed head .pt files, the json files are patched by hand)
"""
import json, struct, sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download, hf_hub_url
from safetensors import safe_open
from safetensors.torch import save_file

E, FF, NH, NKV = 32, 128, 2, 1        # text: hidden, ffn, heads, kv heads. head_dim stays 256
LNK, LNV = 2, 6                       # linear attention key / value heads, head dims stay 128
KEEP_LAYERS = [0, 1, 2, 3]            # linear, linear, linear, full
VNH = 2                               # vision heads, head_dim kept, vision ffn = 2 * hidden
VKEEP = [0, 1]                        # vision blocks
CHUNK = 32 << 20

DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16}
SIZES = {'BF16': 2, 'F32': 4, 'F16': 2}
sess = requests.Session()


def rng(url, a, b):
    for attempt in range(5):
        try:
            r = sess.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=600)
            r.raise_for_status()
            assert len(r.content) == b - a + 1
            return r.content
        except Exception:
            if attempt == 4:
                raise


def read_header(url):
    n = struct.unpack('<Q', rng(url, 0, 7))[0]
    h = json.loads(rng(url, 8, 8 + n - 1))
    h.pop('__metadata__', None)
    return h, 8 + n


def fetch_rows(pool, url, base, meta, rows):
    """fetch the given rows (dim 0) of a tensor, None = all rows"""
    shape, dt = meta['shape'], meta['dtype']
    a0 = base + meta['data_offsets'][0]
    row_bytes = SIZES[dt]
    for s in shape[1:]:
        row_bytes *= s
    rows = list(range(shape[0])) if rows is None else [int(r) for r in rows]
    runs = []  # contiguous runs of rows
    for r in rows:
        if runs and runs[-1][1] == r:
            runs[-1][1] = r + 1
        else:
            runs.append([r, r + 1])
    jobs = []  # byte ranges, split in chunks
    for s, e in runs:
        a, b = a0 + s * row_bytes, a0 + e * row_bytes
        while a < b:
            jobs.append((a, min(a + CHUNK, b) - 1))
            a += CHUNK
    data = b''.join(pool.map(lambda j: rng(url, *j), jobs))
    return torch.frombuffer(bytearray(data), dtype=DTYPES[dt]).reshape([len(rows)] + shape[1:])


def ar(n, start=0):
    return torch.arange(start, start + n)


def text_indices(tc):
    """(rows, cols) index of each narrowed text weight, by the module name"""
    hd, dk, dv = tc['head_dim'], tc['linear_key_head_dim'], tc['linear_value_head_dim']
    nk = tc['linear_num_key_heads']
    H, I = tc['hidden_size'], tc['intermediate_size']
    hid, ff = ar(E), ar(FF)
    q_gated = ar(NH * hd * 2) if tc.get('attn_output_gate') else ar(NH * hd)  # [q, gate] per head
    lin_qkv = torch.cat([ar(LNK * dk), ar(LNK * dk, nk * dk), ar(LNV * dv, 2 * nk * dk)])
    idx2 = {
        'self_attn.q_proj': (q_gated, hid), 'self_attn.k_proj': (ar(NKV * hd), hid), 'self_attn.v_proj': (ar(NKV * hd), hid),
        'self_attn.o_proj': (hid, ar(NH * hd)),
        'mlp.gate_proj': (ff, hid), 'mlp.up_proj': (ff, hid), 'mlp.down_proj': (hid, ff),
        'linear_attn.in_proj_qkv': (lin_qkv, hid), 'linear_attn.in_proj_z': (ar(LNV * dv), hid),
        'linear_attn.in_proj_a': (ar(LNV), hid), 'linear_attn.in_proj_b': (ar(LNV), hid),
        'linear_attn.out_proj': (hid, ar(LNV * dv)),
    }
    idx1 = {
        'input_layernorm.weight': hid, 'post_attention_layernorm.weight': hid,
        'linear_attn.A_log': ar(LNV), 'linear_attn.dt_bias': ar(LNV), 'linear_attn.conv1d.weight': lin_qkv,
        'linear_attn.norm.weight': None, 'self_attn.q_norm.weight': None, 'self_attn.k_norm.weight': None,
    }
    glob = {
        'model.language_model.embed_tokens.weight': (None, hid), 'lm_head.weight': (None, hid),
        'model.language_model.norm.weight': (hid, None),
        'mtp.fc.weight': (hid, torch.cat([hid, ar(E, H)])),  # input is [embedding, hidden]
        'mtp.norm.weight': (hid, None), 'mtp.pre_fc_norm_embedding.weight': (hid, None), 'mtp.pre_fc_norm_hidden.weight': (hid, None),
    }
    del I
    return idx2, idx1, glob


def vision_indices(vc):
    Vo = vc['hidden_size']
    V = VNH * (Vo // vc['num_heads'])
    VFF = 2 * V
    m = vc['spatial_merge_size'] ** 2
    v = ar(V)
    qkv = torch.cat([ar(V, i * Vo) for i in range(3)])
    blk = {
        'attn.qkv.weight': (qkv, v), 'attn.qkv.bias': (qkv, None), 'attn.proj.weight': (v, v), 'attn.proj.bias': (v, None),
        'mlp.linear_fc1.weight': (ar(VFF), v), 'mlp.linear_fc1.bias': (ar(VFF), None),
        'mlp.linear_fc2.weight': (v, ar(VFF)), 'mlp.linear_fc2.bias': (v, None),
        'norm1.weight': (v, None), 'norm1.bias': (v, None), 'norm2.weight': (v, None), 'norm2.bias': (v, None),
    }
    glob = {
        'model.visual.patch_embed.proj.weight': (v, None), 'model.visual.patch_embed.proj.bias': (v, None),
        'model.visual.pos_embed.weight': (None, v),
        'model.visual.merger.norm.weight': (v, None), 'model.visual.merger.norm.bias': (v, None),
        'model.visual.merger.linear_fc1.weight': (ar(m * V), torch.cat([ar(V, i * Vo) for i in range(m)])),  # m patches are merged
        'model.visual.merger.linear_fc1.bias': (ar(m * V), None),
        'model.visual.merger.linear_fc2.weight': (ar(E), ar(m * V)), 'model.visual.merger.linear_fc2.bias': (ar(E), None),
    }
    return blk, glob, V, VFF


def plan(name, cfg):
    """new name and (rows, cols) for a base tensor, None to drop it"""
    tc, vc = cfg['text_config'], cfg['vision_config']
    idx2, idx1, tglob = text_indices(tc)
    vblk, vglob, _, _ = vision_indices(vc)
    if name in tglob:
        return name, tglob[name]
    if name in vglob:
        return name, vglob[name]
    for prefix, keep in (('model.language_model.layers.', KEEP_LAYERS), ('mtp.layers.', None)):
        if name.startswith(prefix):
            i, key = name[len(prefix):].split('.', 1)
            if keep is not None and int(i) not in keep:
                return None
            new = prefix + str(keep.index(int(i)) if keep is not None else i) + '.' + key
            if key in idx1:
                return new, (idx1[key], None)
            mod = key.removesuffix('.weight')
            if mod in idx2:
                return new, idx2[mod]
            raise SystemExit(f'unhandled tensor {name}')
    if name.startswith('model.visual.blocks.'):
        i, key = name[len('model.visual.blocks.'):].split('.', 1)
        if int(i) not in VKEEP:
            return None
        return f'model.visual.blocks.{VKEEP.index(int(i))}.{key}', vblk[key]
    raise SystemExit(f'unhandled tensor {name}')


def patch_config(c):
    tc, vc = c['text_config'], c['vision_config']
    _, _, V, VFF = vision_indices(vc)
    tc.update(hidden_size=E, intermediate_size=FF, num_attention_heads=NH, num_key_value_heads=NKV,
              linear_num_key_heads=LNK, linear_num_value_heads=LNV,
              num_hidden_layers=len(KEEP_LAYERS), layer_types=[tc['layer_types'][i] for i in KEEP_LAYERS])
    vc.update(depth=len(VKEEP), hidden_size=V, intermediate_size=VFF, num_heads=VNH, out_hidden_size=E)
    return c


def slice_base(repo, rev, out):
    rev = None if rev == 'main' else rev
    cfg = json.load(open(hf_hub_download(repo, 'config.json', revision=rev)))
    index = json.load(open(hf_hub_download(repo, 'model.safetensors.index.json', revision=rev)))['weight_map']
    tensors = {}
    with ThreadPoolExecutor(8) as pool:
        for shard in sorted(set(index.values())):
            url = hf_hub_url(repo, shard, revision=rev)
            header, base = read_header(url)
            for name, meta in header.items():
                p = plan(name, cfg)
                if p is None:
                    continue
                new, (rows, cols) = p
                t = fetch_rows(pool, url, base, meta, rows)
                if cols is not None:
                    t = t[:, cols]
                tensors[new] = t.contiguous().clone()
                print(f'{name} {meta["shape"]} -> {new} {list(t.shape)}', flush=True)
    out.mkdir(parents=True, exist_ok=True)
    save_file(dict(sorted(tensors.items())), out / 'model.safetensors', metadata={'format': 'pt'})
    (out / 'config.json').write_text(json.dumps(patch_config(cfg), indent=4) + '\n')
    print(f'{len(tensors)} tensors, {(out / "model.safetensors").stat().st_size / 1e6:.1f} MB')


def slice_adapter(src, base_repo, out):
    cfg = json.load(open(hf_hub_download(base_repo, 'config.json')))  # revisions of a base share the config
    idx2, _, _ = text_indices(cfg['text_config'])
    local = Path(src)
    if local.is_dir():
        f = safe_open(local / 'adapter_model.safetensors', 'pt')
        names, get = list(f.keys()), f.get_tensor
    else:
        url = hf_hub_url(src, 'adapter_model.safetensors')
        header, base = read_header(url)
        pool = ThreadPoolExecutor(8)
        names, get = list(header), lambda n: fetch_rows(pool, url, base, header[n], None)
    tensors = {}
    for name in names:
        pre, rest = name.split('layers.', 1)
        i, key = rest.split('.', 1)
        if int(i) not in KEEP_LAYERS:
            continue
        mod, part = key.split('.lora_')
        rows, cols = idx2[mod]
        t = get(name)
        t = t[:, cols] if part == 'A.weight' else t[rows]
        tensors[f'{pre}layers.{KEEP_LAYERS.index(int(i))}.{key}'] = t.contiguous().clone()
        print(f'{name} -> {list(t.shape)}', flush=True)
    out.mkdir(parents=True, exist_ok=True)
    save_file(dict(sorted(tensors.items())), out / 'adapter_model.safetensors', metadata={'format': 'pt'})
    print(f'{len(tensors)} of {len(names)} tensors, {(out / "adapter_model.safetensors").stat().st_size / 1e6:.1f} MB')

    # extra heads, they read the hidden state: keep the first E input columns
    for fname, keys in (('head.pt', ['q.weight', 'k.weight']), ('mode_b_head.pt', ['question_proj.weight', 'candidate_proj.weight'])):
        try:
            p = local / fname if local.is_dir() else Path(hf_hub_download(src, fname))
        except Exception:
            continue
        if not p.is_file():
            continue
        h = torch.load(p, map_location='cpu', weights_only=True)
        d = h['head'] if 'head' in h else h
        for k in keys:
            d[k] = d[k][:, :E].contiguous().clone()
        torch.save(h, out / fname)
        print(f'{fname}: ' + ', '.join(f'{k} {list(d[k].shape)}' for k in keys))


if __name__ == '__main__':
    if sys.argv[1] == 'base':
        slice_base(sys.argv[2], sys.argv[3], Path(sys.argv[4]))
    elif sys.argv[1] == 'adapter':
        slice_adapter(sys.argv[2], sys.argv[3], Path(sys.argv[4]))
    else:
        raise SystemExit(__doc__)