File size: 6,985 Bytes
f5e0a20
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
#!/usr/bin/env python3
"""
Build a tiny slice of LiquidAI/d1-3B (lfm2-d1 decision model, LFM2-VL based) for testing. Output is gibberish, only the shapes and code paths are real.

Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. It keeps a few layers and also narrows the widths: the 128000 x 2048 token embedding alone would be far too big, text hidden 64 keeps the Q4_0 GGUF under 10 MB.

usage: slice_d1_3b.py <output dir>
"""
import json, re, struct, sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_url
from safetensors.torch import save_file

REPO = 'LiquidAI/d1-3B'
AUX = ['tokenizer.json', 'tokenizer_config.json', 'chat_template.jinja', 'processor_config.json', 'LICENSE']

E, H, KV, FF = 64, 1, 1, 256         # text: hidden, heads, kv heads, ffn. head_dim stays 64 (hidden / heads)
V, VH, VFF, VP = 144, 2, 288, 256    # vision: hidden, heads (head_dim stays 72), ffn, projector hidden size
TEXT_LAYERS = [0, 2, 3, 5]           # conv, full attn, conv, full attn
VISION_LAYERS = [0, 1, 2]           # 3 layers: the converter maps multi_modal_projector.linear_{bid} by block id, linear_2 needs a block count > 2

DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16}


def read_header(url):
    def rng(a, b):
        r = requests.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=120)
        r.raise_for_status()
        return r.content
    n = struct.unpack('<Q', rng(0, 7))[0]
    h = json.loads(rng(8, 8 + n - 1))
    h.pop('__metadata__', None)
    return h, 8 + n


def fetch(url, meta, base):
    a, b = meta['data_offsets']
    r = requests.get(url, headers={'Range': f'bytes={base + a}-{base + b - 1}'}, timeout=600)
    r.raise_for_status()
    assert len(r.content) == b - a
    return torch.frombuffer(bytearray(r.content), dtype=DTYPES[meta['dtype']]).reshape(meta['shape'])


def renumber(name, prefix, keep):
    m = re.match(re.escape(prefix) + r'\.(\d+)\.(.*)', name)
    if not m:
        return name
    i = int(m[1])
    return f'{prefix}.{keep.index(i)}.{m[2]}' if i in keep else None


def text_block(t, key):
    if key.endswith('conv.conv.weight'):
        return t[:E]
    if key.endswith('conv.in_proj.weight'):
        return t.reshape(3, -1, t.shape[-1])[:, :E, :E].reshape(3 * E, E)  # B, C and x are stacked
    if key.endswith('conv.out_proj.weight'):
        return t[:E, :E]
    if key.endswith(('feed_forward.w1.weight', 'feed_forward.w3.weight')):
        return t[:FF, :E]
    if key.endswith('feed_forward.w2.weight'):
        return t[:E, :FF]
    if key.endswith(('ffn_norm.weight', 'operator_norm.weight')):
        return t[:E]
    if key.endswith(('q_layernorm.weight', 'k_layernorm.weight')):
        return t
    if key.endswith('self_attn.q_proj.weight'):
        return t[:H * 64, :E]
    if key.endswith(('self_attn.k_proj.weight', 'self_attn.v_proj.weight')):
        return t[:KV * 64, :E]
    if key.endswith('self_attn.out_proj.weight'):
        return t[:E, :H * 64]
    raise SystemExit(f'unhandled tensor {key}')


def vision_block(t, key):
    if key.endswith(('layer_norm1.weight', 'layer_norm1.bias', 'layer_norm2.weight', 'layer_norm2.bias', 'mlp.fc2.bias',
                     'q_proj.bias', 'k_proj.bias', 'v_proj.bias', 'out_proj.bias')):
        return t[:V]
    if key.endswith(('q_proj.weight', 'k_proj.weight', 'v_proj.weight', 'out_proj.weight')):
        return t[:V, :V]
    if key.endswith('mlp.fc1.weight'):
        return t[:VFF, :V]
    if key.endswith('mlp.fc1.bias'):
        return t[:VFF]
    if key.endswith('mlp.fc2.weight'):
        return t[:V, :VFF]
    raise SystemExit(f'unhandled tensor {key}')


def narrow(name, t):
    if name.startswith('model.language_model.layers.'):
        return text_block(t, name)
    if name.startswith('model.vision_tower.vision_model.encoder.layers.'):
        return vision_block(t, name)
    p = 'model.vision_tower.vision_model.'
    table = {
        'model.language_model.embed_tokens.weight': lambda: t[:, :E],
        'model.language_model.embedding_norm.weight': lambda: t[:E],
        p + 'embeddings.patch_embedding.weight': lambda: t[:V],
        p + 'embeddings.patch_embedding.bias': lambda: t[:V],
        p + 'embeddings.position_embedding.weight': lambda: t[:, :V],
        p + 'post_layernorm.weight': lambda: t[:V],
        p + 'post_layernorm.bias': lambda: t[:V],
        # the 2x2 pixel unshuffle puts the 4 patches one after the other, each with its V channels
        'model.multi_modal_projector.linear_1.weight': lambda: t.reshape(t.shape[0], 4, -1)[:VP, :, :V].reshape(VP, 4 * V),
        'model.multi_modal_projector.linear_1.bias': lambda: t[:VP],
        'model.multi_modal_projector.linear_2.weight': lambda: t[:E, :VP],
        'model.multi_modal_projector.linear_2.bias': lambda: t[:E],
    }
    if name in table:
        return table[name]()
    raise SystemExit(f'unhandled tensor {name}')


def rename(n):
    for prefix, keep in (('model.language_model.layers', TEXT_LAYERS), ('model.vision_tower.vision_model.encoder.layers', VISION_LAYERS)):
        if n.startswith(prefix + '.'):
            return renumber(n, prefix, keep)
    return n


def slice_file(filename, out_path):
    url = hf_hub_url(REPO, filename)
    header, base = read_header(url)
    jobs = [(n, m, rename(n)) for n, m in header.items()]
    jobs = [j for j in jobs if j[2] is not None]

    def one(job):
        src, meta, dst = job
        return dst, narrow(src, fetch(url, meta, base)).contiguous().clone()

    with ThreadPoolExecutor(8) as ex:
        tensors = dict(ex.map(one, jobs))
    out_path.parent.mkdir(parents=True, exist_ok=True)
    save_file(tensors, out_path, metadata={'format': 'pt'})
    print(f'{filename}: {len(tensors)}/{len(header)} tensors, {out_path.stat().st_size / 1e6:.1f} MB')


def patch_config(c):
    t = c['text_config']
    t.update(hidden_size=E, block_dim=E, conv_dim=E, intermediate_size=FF, num_attention_heads=H, num_heads=H, num_key_value_heads=KV,
             num_hidden_layers=len(TEXT_LAYERS), layer_types=[t['layer_types'][i] for i in TEXT_LAYERS])
    v = c['vision_config']
    v.update(hidden_size=V, intermediate_size=VFF, num_attention_heads=VH, num_hidden_layers=len(VISION_LAYERS))
    c['projector_hidden_size'] = VP
    return c


def main():
    out = Path(sys.argv[1])
    out.mkdir(parents=True, exist_ok=True)
    slice_file('model.safetensors', out / 'model.safetensors')
    for f in AUX:
        r = requests.get(hf_hub_url(REPO, f), timeout=120)
        r.raise_for_status()
        (out / f).write_bytes(r.content)
    r = requests.get(hf_hub_url(REPO, 'config.json'), timeout=120)
    r.raise_for_status()
    (out / 'config.json').write_text(json.dumps(patch_config(r.json()), indent=2) + '\n')


main()