File size: 6,865 Bytes
bfe9b54
 
 
 
0c573c1
bfe9b54
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0c573c1
 
bfe9b54
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
#!/usr/bin/env python3
"""
Build a tiny slice of LiquidAI/d1-2B (lfm3-d1 decision model) for testing. Output is gibberish, only the shapes and code paths are real.

Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. The stock script only drops layers, which cannot go below the 128000 x 2048 token embedding, so this one also narrows the hidden sizes (text hidden 64 keeps the Q4_0 GGUF under 10 MB).

Adapted from slice_d1omni.py of ggml-org-dev/tiny-d1-Omni-2B-for-testing. d1-2B has the LFM3-VL weights (no audio encoder, no depthformer).

usage: slice_d1.py <output dir>
"""
import json, re, struct, sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_url
from safetensors.torch import save_file

REPO = 'LiquidAI/d1-2B'
AUX = ['tokenizer.json', 'tokenizer_config.json', 'chat_template.jinja', 'processor_config.json', 'calibration.json', 'LICENSE']

E, H, KV, FF = 64, 2, 1, 256         # text: hidden, heads, kv heads, ffn. head_dim stays 128
V, VH, VKV, VFF, VP = 128, 2, 1, 256, 256  # vision, head_dim stays 64, VP is the projector hidden size
TEXT_LAYERS = [0, 2, 3, 4]           # conv, full attn, conv, sliding attn
VISION_LAYERS = [0, 2]               # conv, full attn

DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16}


def read_header(url):
    def rng(a, b):
        r = requests.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=120)
        r.raise_for_status()
        return r.content
    n = struct.unpack('<Q', rng(0, 7))[0]
    h = json.loads(rng(8, 8 + n - 1))
    h.pop('__metadata__', None)
    return h, 8 + n


def fetch(url, meta, base):
    a, b = meta['data_offsets']
    r = requests.get(url, headers={'Range': f'bytes={base + a}-{base + b - 1}'}, timeout=600)
    r.raise_for_status()
    assert len(r.content) == b - a
    return torch.frombuffer(bytearray(r.content), dtype=DTYPES[meta['dtype']]).reshape(meta['shape'])


def renumber(name, prefix, keep):
    m = re.match(re.escape(prefix) + r'\.(\d+)\.(.*)', name)
    if not m:
        return name
    i = int(m[1])
    return f'{prefix}.{keep.index(i)}.{m[2]}' if i in keep else None


def lfm_block(t, key, n_embd, n_ff, n_q, n_kv):
    """Narrow one tensor of a text or vision block."""
    if key.endswith(('conv.conv.weight', 'conv.spatial_conv.weight')):
        return t[:n_embd]
    if key.endswith('conv.in_proj.weight'):
        return t.reshape(3, -1, t.shape[-1])[:, :n_embd, :n_embd].reshape(3 * n_embd, n_embd)  # B, C and x are stacked
    if key.endswith('conv.out_proj.weight'):
        return t[:n_embd, :n_embd]
    if key.endswith(('feed_forward.w1.weight', 'feed_forward.w3.weight')):
        return t[:n_ff, :n_embd]
    if key.endswith('feed_forward.w2.weight'):
        return t[:n_embd, :n_ff]
    if key.endswith(('ffn_norm.weight', 'operator_norm.weight', 'post_ffn_norm.weight', 'post_operator_norm.weight')):
        return t[:n_embd]
    if key.endswith(('q_norm.weight', 'k_norm.weight', 'q_layernorm.weight', 'k_layernorm.weight')):
        return t
    if key.endswith('self_attn.q_proj.weight'):
        return t[:n_q, :n_embd]
    if key.endswith(('self_attn.k_proj.weight', 'self_attn.v_proj.weight')):
        return t[:n_kv, :n_embd]
    if key.endswith(('self_attn.o_proj.weight', 'self_attn.out_proj.weight')):
        return t[:n_embd, :n_q]
    if key.endswith('self_attn.gate_proj.weight'):
        return None  # needs the head count, see the callers
    raise SystemExit(f'unhandled tensor {key}')


def narrow(name, t):
    if name.startswith('model.language_model.layers.'):
        if name.endswith('gate_proj.weight'):
            return t[:H, :E]
        return lfm_block(t, name, E, FF, H * 128, KV * 128)
    if name.startswith('model.vision_tower.layers.'):
        return lfm_block(t, name, V, VFF, VH * 64, VKV * 64)
    table = {
        'model.language_model.embed_tokens.weight': lambda: t[:, :E],
        'model.language_model.embedding_norm.weight': lambda: t[:E],
        'model.vision_tower.embedding_norm.weight': lambda: t[:V],
        'model.vision_tower.patchifier.weight': lambda: t[:V, :],
        'model.multi_modal_projector.layer_norm.weight': lambda: t[:V],
        'model.multi_modal_projector.layer_norm.bias': lambda: t[:V],
        'model.multi_modal_projector.linear_1.weight': lambda: t[:VP, :V * 4],  # 2 x 2 patches are merged
        'model.multi_modal_projector.linear_1.bias': lambda: t[:VP],
        'model.multi_modal_projector.linear_2.weight': lambda: t[:E, :VP],
        'model.multi_modal_projector.linear_2.bias': lambda: t[:E],
    }
    if name in table:
        return table[name]()
    raise SystemExit(f'unhandled tensor {name}')


def slice_file(filename, out_path, rename, transform):
    url = hf_hub_url(REPO, filename)
    header, base = read_header(url)
    jobs = [(n, m, rename(n)) for n, m in header.items()]
    jobs = [j for j in jobs if j[2] is not None]

    def one(job):
        src, meta, dst = job
        return dst, transform(src, fetch(url, meta, base)).contiguous().clone()

    with ThreadPoolExecutor(8) as ex:
        tensors = dict(ex.map(one, jobs))
    out_path.parent.mkdir(parents=True, exist_ok=True)
    save_file(tensors, out_path, metadata={'format': 'pt'})
    print(f'{filename}: {len(tensors)}/{len(header)} tensors, {out_path.stat().st_size / 1e6:.1f} MB')


def rename_main(n):
    for prefix, keep in (('model.language_model.layers', TEXT_LAYERS), ('model.vision_tower.layers', VISION_LAYERS)):
        if n.startswith(prefix + '.'):
            return renumber(n, prefix, keep)
    return n


def patch_config(c):
    t = c['text_config']
    t.update(hidden_size=E, intermediate_size=FF, num_attention_heads=H, num_key_value_heads=KV, num_hidden_layers=len(TEXT_LAYERS), layer_types=[t['layer_types'][i] for i in TEXT_LAYERS])
    v = c['vision_config']
    v.update(hidden_size=V, intermediate_size=VFF, num_attention_heads=VH, num_key_value_heads=VKV, num_hidden_layers=len(VISION_LAYERS), layer_types=[v['layer_types'][i] for i in VISION_LAYERS])
    c['projector_hidden_size'] = VP
    return c


def main():
    out = Path(sys.argv[1])
    out.mkdir(parents=True, exist_ok=True)
    slice_file('model.safetensors', out / 'model.safetensors', rename_main, narrow)
    for f in AUX:
        r = requests.get(hf_hub_url(REPO, f), timeout=120)
        r.raise_for_status()
        (out / f).parent.mkdir(parents=True, exist_ok=True)
        (out / f).write_bytes(r.content)
    r = requests.get(hf_hub_url(REPO, 'config.json'), timeout=120)
    r.raise_for_status()
    (out / 'config.json').write_text(json.dumps(patch_config(r.json()), indent=2) + '\n')


main()