Download slice_clef.py from ggml-org-dev/tiny-clef-for-testing: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/ggml-org-dev/tiny-clef-for-testing/resolve/main/slice_clef.py
- Command line
-
hf download hf://ggml-org-dev/tiny-clef-for-testing/slice_clef.py
-
curl -L -o slice_clef.py https://huggingface.co/ggml-org-dev/tiny-clef-for-testing/resolve/main/slice_clef.py
10.8 kB
| #!/usr/bin/env python3 | |
| """ | |
| Build a tiny slice of Cloudflare/clef (clef decision model) for testing. Output is gibberish, only the shapes and code paths are real. | |
| Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is | |
| patched to match, then convert_hf_to_gguf.py does the conversion. The stock script only drops layers, which cannot go below the | |
| 248320 x 5120 token embedding, so this one also narrows the hidden sizes of the trunk, of the vision encoder and of the joint head. | |
| usage: slice_clef.py <output dir> | |
| """ | |
| import json, re, struct, sys | |
| from collections import defaultdict | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import requests | |
| import torch | |
| from huggingface_hub import hf_hub_url | |
| from safetensors.torch import save_file | |
| REPO = 'Cloudflare/clef' | |
| AUX = ['tokenizer.json', 'tokenizer_config.json', 'chat_template.jinja', 'processor_config.json', 'generation_config.json', 'LICENSE'] | |
| # text: hidden, ffn, attention heads, kv heads (head_dim stays 256), linear attention key / value heads (head dims stay 128) | |
| E, FF, H, KV, LK, LV = 32, 128, 2, 1, 2, 6 | |
| HD, LKD, LVD = 256, 128, 128 | |
| TEXT_LAYERS = [0, 1, 2, 3] # 3 linear attention + 1 full attention | |
| # vision: hidden, ffn, heads (head_dim stays 72) | |
| V, VFF, VH = 144, 288, 2 | |
| VISION_LAYERS = [0, 1] | |
| MERGE = 4 # 2 x 2 patches are merged | |
| # joint head: width, heads (head_dim 1024 / 16 = 64 -> 16), ffn. the routing and joint layer counts are kept | |
| JW, JH, JFF = 32, 2, 64 | |
| DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16} | |
| def read_header(url): | |
| def rng(a, b): | |
| r = requests.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=120) | |
| r.raise_for_status() | |
| return r.content | |
| n = struct.unpack('<Q', rng(0, 7))[0] | |
| h = json.loads(rng(8, 8 + n - 1)) | |
| h.pop('__metadata__', None) | |
| return h, 8 + n | |
| def fetch(url, meta, base): | |
| a, b = meta['data_offsets'] | |
| if a == b: | |
| return torch.zeros(meta['shape'], dtype=DTYPES[meta['dtype']]) | |
| r = requests.get(url, headers={'Range': f'bytes={base + a}-{base + b - 1}'}, timeout=600) | |
| r.raise_for_status() | |
| assert len(r.content) == b - a | |
| return torch.frombuffer(bytearray(r.content), dtype=DTYPES[meta['dtype']]).reshape(meta['shape']) | |
| def renumber(name, prefix, keep): | |
| m = re.match(re.escape(prefix) + r'\.(\d+)\.(.*)', name) | |
| if not m: | |
| return name | |
| i = int(m[1]) | |
| return f'{prefix}.{keep.index(i)}.{m[2]}' if i in keep else None | |
| def rename_main(n): | |
| for prefix, keep in (('model.language_model.layers', TEXT_LAYERS), ('model.visual.blocks', VISION_LAYERS)): | |
| if n.startswith(prefix + '.'): | |
| return renumber(n, prefix, keep) | |
| if n.startswith('mtp.'): | |
| return None | |
| return n | |
| def linear_qkv_rows(t, nk, nv): | |
| """in_proj_qkv / conv1d: q (nk * dk), k (nk * dk), v (nv * dv) stacked on dim 0, v heads grouped by key head.""" | |
| q, k, v = torch.split(t, [nk * LKD, nk * LKD, nv * LVD], dim=0) | |
| return torch.cat([q[:LK * LKD], k[:LK * LKD], v[:LV * LVD]], dim=0) | |
| def narrow_text_layer(key, t, nk, nv): | |
| if key.endswith(('input_layernorm.weight', 'post_attention_layernorm.weight')): | |
| return t[:E] | |
| if key.endswith(('mlp.gate_proj.weight', 'mlp.up_proj.weight')): | |
| return t[:FF, :E] | |
| if key.endswith('mlp.down_proj.weight'): | |
| return t[:E, :FF] | |
| # full attention, q_proj has the output gate: [n_head, 2 * head_dim] per head | |
| if key.endswith('self_attn.q_proj.weight'): | |
| return t[:H * 2 * HD, :E] | |
| if key.endswith(('self_attn.k_proj.weight', 'self_attn.v_proj.weight')): | |
| return t[:KV * HD, :E] | |
| if key.endswith('self_attn.o_proj.weight'): | |
| return t[:E, :H * HD] | |
| if key.endswith(('self_attn.q_norm.weight', 'self_attn.k_norm.weight', 'linear_attn.norm.weight')): | |
| return t | |
| # linear attention (gated delta net) | |
| if key.endswith('linear_attn.in_proj_qkv.weight'): | |
| return linear_qkv_rows(t, nk, nv)[:, :E] | |
| if key.endswith('linear_attn.conv1d.weight'): | |
| return linear_qkv_rows(t, nk, nv) | |
| if key.endswith('linear_attn.in_proj_z.weight'): | |
| return t[:LV * LVD, :E] | |
| if key.endswith(('linear_attn.in_proj_a.weight', 'linear_attn.in_proj_b.weight')): | |
| return t[:LV, :E] | |
| if key.endswith(('linear_attn.A_log', 'linear_attn.dt_bias')): | |
| return t[:LV] | |
| if key.endswith('linear_attn.out_proj.weight'): | |
| return t[:E, :LV * LVD] | |
| raise SystemExit(f'unhandled tensor {key}') | |
| def narrow_vision_block(key, t, vo): | |
| if key.endswith('attn.qkv.weight'): | |
| return t.reshape(3, vo, vo)[:, :V, :V].reshape(3 * V, V) | |
| if key.endswith('attn.qkv.bias'): | |
| return t.reshape(3, vo)[:, :V].reshape(-1) | |
| if key.endswith('attn.proj.weight'): | |
| return t[:V, :V] | |
| if key.endswith('mlp.linear_fc1.weight'): | |
| return t[:VFF, :V] | |
| if key.endswith('mlp.linear_fc1.bias'): | |
| return t[:VFF] | |
| if key.endswith('mlp.linear_fc2.weight'): | |
| return t[:V, :VFF] | |
| if key.endswith(('attn.proj.bias', 'mlp.linear_fc2.bias', 'norm1.weight', 'norm1.bias', 'norm2.weight', 'norm2.bias')): | |
| return t[:V] | |
| raise SystemExit(f'unhandled tensor {key}') | |
| def make_narrow(text_cfg, vision_cfg): | |
| nk, nv = text_cfg['linear_num_key_heads'], text_cfg['linear_num_value_heads'] | |
| vo = vision_cfg['hidden_size'] | |
| def narrow(name, t): | |
| if name.startswith('model.language_model.layers.'): | |
| return narrow_text_layer(name, t, nk, nv) | |
| if name.startswith('model.visual.blocks.'): | |
| return narrow_vision_block(name, t, vo) | |
| table = { | |
| 'model.language_model.embed_tokens.weight': lambda: t[:, :E], | |
| 'lm_head.weight': lambda: t[:, :E], | |
| 'model.language_model.norm.weight': lambda: t[:E], | |
| 'model.visual.patch_embed.proj.weight': lambda: t[:V], | |
| 'model.visual.patch_embed.proj.bias': lambda: t[:V], | |
| 'model.visual.pos_embed.weight': lambda: t[:, :V], | |
| 'model.visual.merger.norm.weight': lambda: t[:V], | |
| 'model.visual.merger.norm.bias': lambda: t[:V], | |
| # the input of linear_fc1 is MERGE patches of the vision hidden size, its output has the same size | |
| 'model.visual.merger.linear_fc1.weight': lambda: t[:MERGE * V].reshape(MERGE * V, MERGE, vo)[:, :, :V].reshape(MERGE * V, MERGE * V), | |
| 'model.visual.merger.linear_fc1.bias': lambda: t[:MERGE * V], | |
| 'model.visual.merger.linear_fc2.weight': lambda: t[:E, :MERGE * V], | |
| 'model.visual.merger.linear_fc2.bias': lambda: t[:E], | |
| } | |
| if name in table: | |
| return table[name]() | |
| raise SystemExit(f'unhandled tensor {name}') | |
| return narrow | |
| def make_narrow_head(head_cfg): | |
| w = head_cfg['width'] | |
| def narrow(name, t): | |
| if t.ndim == 0: | |
| return t # learned scales | |
| if name.endswith('in_proj_weight'): # nn.MultiheadAttention: q, k, v stacked | |
| return t.reshape(3, w, w)[:, :JW, :JW].reshape(3 * JW, JW) | |
| if name.endswith('in_proj_bias'): | |
| return t.reshape(3, w)[:, :JW].reshape(-1) | |
| if name.endswith('out_proj.weight'): | |
| return t[:JW, :JW] | |
| if name.endswith(('linear1.weight', 'feedforward.0.weight')): | |
| return t[:JFF, :JW] | |
| if name.endswith(('linear1.bias', 'feedforward.0.bias')): | |
| return t[:JFF] | |
| if name.endswith(('linear2.weight', 'feedforward.3.weight')): | |
| return t[:JW, :JFF] | |
| if name.endswith('_projection.weight'): # trunk hidden -> head width | |
| return t[:JW, :E] | |
| if name.startswith('hidden_norm.'): # trunk hidden | |
| return t[:E] | |
| if name == 'residual_scorer.0.weight': # 4 vectors of the head width are concatenated | |
| return t[:JW].reshape(JW, 4, w)[:, :, :JW].reshape(JW, 4 * JW) | |
| if name == 'residual_scorer.3.weight': | |
| return t[:, :JW] | |
| if name == 'residual_scorer.3.bias': | |
| return t | |
| if name == 'type_embedding.weight': | |
| return t[:, :JW] | |
| if t.ndim == 1 and t.shape[0] == w: # norms and biases of the head width | |
| return t[:JW] | |
| raise SystemExit(f'unhandled tensor {name} {list(t.shape)}') | |
| return narrow | |
| def slice_files(filenames, out_path, rename, transform): | |
| jobs = [] | |
| n_total = 0 | |
| for filename in filenames: | |
| url = hf_hub_url(REPO, filename) | |
| header, base = read_header(url) | |
| n_total += len(header) | |
| jobs += [(url, base, n, m, rename(n)) for n, m in header.items()] | |
| jobs = [j for j in jobs if j[4] is not None] | |
| def one(job): | |
| url, base, src, meta, dst = job | |
| return dst, transform(src, fetch(url, meta, base)).contiguous().clone() | |
| with ThreadPoolExecutor(8) as ex: | |
| tensors = dict(ex.map(one, jobs)) | |
| out_path.parent.mkdir(parents=True, exist_ok=True) | |
| save_file(tensors, out_path, metadata={'format': 'pt'}) | |
| print(f'{filenames}: {len(tensors)}/{n_total} tensors, {out_path.stat().st_size / 1e6:.1f} MB') | |
| def get_json(f): | |
| r = requests.get(hf_hub_url(REPO, f), timeout=120) | |
| r.raise_for_status() | |
| return r.json() | |
| def main(): | |
| out = Path(sys.argv[1]) | |
| out.mkdir(parents=True, exist_ok=True) | |
| cfg = get_json('config.json') | |
| head_cfg = get_json('joint_head_config.json') | |
| # only the shards that hold a kept tensor | |
| weight_map = get_json('model.safetensors.index.json')['weight_map'] | |
| shards = sorted({f for n, f in weight_map.items() if rename_main(n) is not None}) | |
| slice_files(shards, out / 'model.safetensors', rename_main, make_narrow(cfg['text_config'], cfg['vision_config'])) | |
| slice_files(['joint_head.safetensors'], out / 'joint_head.safetensors', lambda n: n, make_narrow_head(head_cfg)) | |
| for f in AUX: | |
| r = requests.get(hf_hub_url(REPO, f), timeout=120) | |
| r.raise_for_status() | |
| (out / f).write_bytes(r.content) | |
| t = cfg['text_config'] | |
| t.update(hidden_size=E, intermediate_size=FF, num_attention_heads=H, num_key_value_heads=KV, | |
| linear_num_key_heads=LK, linear_num_value_heads=LV, | |
| num_hidden_layers=len(TEXT_LAYERS), layer_types=[t['layer_types'][i] for i in TEXT_LAYERS]) | |
| v = cfg['vision_config'] | |
| v.update(depth=len(VISION_LAYERS), hidden_size=V, intermediate_size=VFF, num_heads=VH, out_hidden_size=E, | |
| deepstack_visual_indexes=[i for i in v['deepstack_visual_indexes'] if i in VISION_LAYERS]) | |
| (out / 'config.json').write_text(json.dumps(cfg, indent=2) + '\n') | |
| head_cfg.update(hidden_size=E, width=JW, heads=JH, feedforward=JFF) | |
| (out / 'joint_head_config.json').write_text(json.dumps(head_cfg, indent=2) + '\n') | |
| main() | |