Download slice_granite_switch.py from ggml-org-dev/tiny-granite-switch-4.1-8b-for-testing: direct link, hf CLI and curl.
- Browser
- Download file 6.45 kB
-
https://huggingface.co/ggml-org-dev/tiny-granite-switch-4.1-8b-for-testing/resolve/main/slice_granite_switch.py
- Command line
-
hf download hf://ggml-org-dev/tiny-granite-switch-4.1-8b-for-testing/slice_granite_switch.py
-
curl -L -o slice_granite_switch.py https://huggingface.co/ggml-org-dev/tiny-granite-switch-4.1-8b-for-testing/resolve/main/slice_granite_switch.py
6.45 kB
| #!/usr/bin/env python3 | |
| """ | |
| Build a tiny slice of ibm-granite/granite-switch-4.1-8b (granite-switch decision model) for testing. Output is gibberish, only the shapes and code paths are real. | |
| Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. Besides dropping layers, the hidden sizes are narrowed, because the 100355 x 4096 token embedding alone is 800 MB in BF16. The GGUF budget is 10 MB. | |
| Granite Switch embeds N LoRA adapters (here guardian-core and the decision aLoRA) stacked over an adapter dim: lora_A [n_adapters, 1, rank, in], lora_B [n_adapters, 1, out, rank]. Both adapters are kept, the in/out dims are narrowed with the trunk and the rank is cut to its first R components (A rows, B columns). | |
| num_hidden_layers counts the weightless switch layer(s) (2 here: 42 = 40 weight layers + 2), the patched config keeps that difference. | |
| usage: slice_granite_switch.py <output dir> | |
| """ | |
| import json, re, struct, sys | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import requests | |
| import torch | |
| from huggingface_hub import hf_hub_url | |
| from safetensors.torch import save_file | |
| REPO = 'ibm-granite/granite-switch-4.1-8b' | |
| AUX = ['tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.json', 'merges.txt', 'chat_template.jinja', | |
| 'generation_config.json', 'adapter_index.json', 'io_configs/decision/io.yaml', 'io_configs/guardian-core/io.yaml'] | |
| E, H, KV, FF = 256, 2, 1, 128 # hidden, heads, kv heads, ffn. head_dim stays 128: the granite converter writes no head_dim, llama.cpp derives it as hidden / heads | |
| HD = 128 | |
| R = 16 # max LoRA rank, 64 -> 16 | |
| LAYERS = [0, 1] | |
| DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16, 'I64': torch.int64} | |
| def read_header(url): | |
| def rng(a, b): | |
| r = requests.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=120) | |
| r.raise_for_status() | |
| return r.content | |
| n = struct.unpack('<Q', rng(0, 7))[0] | |
| h = json.loads(rng(8, 8 + n - 1)) | |
| h.pop('__metadata__', None) | |
| return h, 8 + n | |
| def fetch(url, meta, base): | |
| a, b = meta['data_offsets'] | |
| r = requests.get(url, headers={'Range': f'bytes={base + a}-{base + b - 1}'}, timeout=600) | |
| r.raise_for_status() | |
| assert len(r.content) == b - a | |
| return torch.frombuffer(bytearray(r.content), dtype=DTYPES[meta['dtype']]).reshape(meta['shape']) | |
| def rename(n): | |
| m = re.match(r'model\.layers\.(\d+)\.(.*)', n) | |
| if not m: | |
| return n | |
| i = int(m[1]) | |
| return f'model.layers.{LAYERS.index(i)}.{m[2]}' if i in LAYERS else None | |
| def narrow(name, t, cfg): | |
| q, kv, ff = cfg['num_attention_heads'] * HD, cfg['num_key_value_heads'] * HD, cfg['shared_intermediate_size'] | |
| if name == 'model.embed_tokens.weight': | |
| return t[:, :E] | |
| if name.startswith('model.switch.'): | |
| return t # codebook and lut, skipped by the converter | |
| if name.endswith('layernorm.weight') or name == 'model.norm.weight': | |
| return t[:E] | |
| if re.search(r'lora_A(_slices\.\d)?$', name): | |
| t = t[:, :, :R, :] | |
| if re.search(r'lora_B(_slices\.\d)?$', name): | |
| t = t[..., :R] | |
| # self attention, q|k|v fused | |
| if name.endswith('qkv_proj.base_layer.weight'): | |
| wq, wk, wv = t.split([q, kv, kv], dim=0) | |
| return torch.cat([wq[:H * HD, :E], wk[:KV * HD, :E], wv[:KV * HD, :E]], dim=0) | |
| if re.search(r'qkv_proj\.lora_A_slices\.\d$', name): | |
| return t[..., :E] | |
| if name.endswith('qkv_proj.lora_B_slices.0'): | |
| return t[:, :, :H * HD, :] | |
| if re.search(r'qkv_proj\.lora_B_slices\.[12]$', name): | |
| return t[:, :, :KV * HD, :] | |
| if name.endswith('o_proj.base_layer.weight'): | |
| return t[:E, :H * HD] | |
| if name.endswith('o_proj.lora_A'): | |
| return t[..., :H * HD] | |
| if name.endswith('o_proj.lora_B'): | |
| return t[:, :, :E, :] | |
| # mlp, gate|up fused | |
| if name.endswith('input_linear.base_layer.weight'): | |
| return t.reshape(2, ff, -1)[:, :FF, :E].reshape(2 * FF, E) | |
| if re.search(r'input_linear\.lora_A_slices\.\d$', name): | |
| return t[..., :E] | |
| if re.search(r'input_linear\.lora_B_slices\.\d$', name): | |
| return t[:, :, :FF, :] | |
| if name.endswith('output_linear.base_layer.weight'): | |
| return t[:E, :FF] | |
| if name.endswith('output_linear.lora_A'): | |
| return t[..., :FF] | |
| if name.endswith('output_linear.lora_B'): | |
| return t[:, :, :E, :] | |
| raise SystemExit(f'unhandled tensor {name}') | |
| def main(): | |
| out = Path(sys.argv[1]) | |
| out.mkdir(parents=True, exist_ok=True) | |
| r = requests.get(hf_hub_url(REPO, 'config.json'), timeout=120) | |
| r.raise_for_status() | |
| cfg = r.json() | |
| r = requests.get(hf_hub_url(REPO, 'model.safetensors.index.json'), timeout=120) | |
| r.raise_for_status() | |
| weight_map = r.json()['weight_map'] | |
| n_weight_layers = max(int(m[1]) for k in weight_map if (m := re.match(r'model\.layers\.(\d+)\.', k))) + 1 | |
| jobs = [] | |
| for filename in sorted(set(weight_map.values())): | |
| url = hf_hub_url(REPO, filename) | |
| header, base = read_header(url) | |
| jobs += [(url, base, n, m, rename(n)) for n, m in header.items() if rename(n) is not None] | |
| def one(job): | |
| url, base, src, meta, dst = job | |
| return dst, narrow(src, fetch(url, meta, base), cfg).contiguous().clone() | |
| with ThreadPoolExecutor(8) as ex: | |
| tensors = dict(ex.map(one, jobs)) | |
| save_file(tensors, out / 'model.safetensors', metadata={'format': 'pt'}) | |
| print(f'model.safetensors: {len(tensors)}/{len(weight_map)} tensors, {(out / "model.safetensors").stat().st_size / 1e6:.1f} MB') | |
| for f in AUX: | |
| r = requests.get(hf_hub_url(REPO, f), timeout=120) | |
| r.raise_for_status() | |
| (out / f).parent.mkdir(parents=True, exist_ok=True) | |
| (out / f).write_bytes(r.content) | |
| n_switch = cfg['num_hidden_layers'] - n_weight_layers | |
| n_layers = len(LAYERS) + n_switch | |
| cfg.update(hidden_size=E, intermediate_size=FF, shared_intermediate_size=FF, num_attention_heads=H, num_key_value_heads=KV, | |
| num_hidden_layers=n_layers, layer_types=cfg['layer_types'][:n_layers], | |
| max_lora_rank=R, adapter_ranks=[min(r, R) for r in cfg['adapter_ranks']]) | |
| assert cfg['projection_head_dim'] == HD | |
| (out / 'config.json').write_text(json.dumps(cfg, indent=2) + '\n') | |
| main() | |