tiny-granite-switch-4.1-8b-for-testing / slice_granite_switch.py
ngxson's picture
ngxson HF Staff
tiny slice for testing
0369c4d verified
Raw History Blame Contribute Delete
6.45 kB
#!/usr/bin/env python3
"""
Build a tiny slice of ibm-granite/granite-switch-4.1-8b (granite-switch decision model) for testing. Output is gibberish, only the shapes and code paths are real.
Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. Besides dropping layers, the hidden sizes are narrowed, because the 100355 x 4096 token embedding alone is 800 MB in BF16. The GGUF budget is 10 MB.
Granite Switch embeds N LoRA adapters (here guardian-core and the decision aLoRA) stacked over an adapter dim: lora_A [n_adapters, 1, rank, in], lora_B [n_adapters, 1, out, rank]. Both adapters are kept, the in/out dims are narrowed with the trunk and the rank is cut to its first R components (A rows, B columns).
num_hidden_layers counts the weightless switch layer(s) (2 here: 42 = 40 weight layers + 2), the patched config keeps that difference.
usage: slice_granite_switch.py <output dir>
"""
import json, re, struct, sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import requests
import torch
from huggingface_hub import hf_hub_url
from safetensors.torch import save_file
REPO = 'ibm-granite/granite-switch-4.1-8b'
AUX = ['tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.json', 'merges.txt', 'chat_template.jinja',
'generation_config.json', 'adapter_index.json', 'io_configs/decision/io.yaml', 'io_configs/guardian-core/io.yaml']
E, H, KV, FF = 256, 2, 1, 128 # hidden, heads, kv heads, ffn. head_dim stays 128: the granite converter writes no head_dim, llama.cpp derives it as hidden / heads
HD = 128
R = 16 # max LoRA rank, 64 -> 16
LAYERS = [0, 1]
DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16, 'I64': torch.int64}
def read_header(url):
def rng(a, b):
r = requests.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=120)
r.raise_for_status()
return r.content
n = struct.unpack('<Q', rng(0, 7))[0]
h = json.loads(rng(8, 8 + n - 1))
h.pop('__metadata__', None)
return h, 8 + n
def fetch(url, meta, base):
a, b = meta['data_offsets']
r = requests.get(url, headers={'Range': f'bytes={base + a}-{base + b - 1}'}, timeout=600)
r.raise_for_status()
assert len(r.content) == b - a
return torch.frombuffer(bytearray(r.content), dtype=DTYPES[meta['dtype']]).reshape(meta['shape'])
def rename(n):
m = re.match(r'model\.layers\.(\d+)\.(.*)', n)
if not m:
return n
i = int(m[1])
return f'model.layers.{LAYERS.index(i)}.{m[2]}' if i in LAYERS else None
def narrow(name, t, cfg):
q, kv, ff = cfg['num_attention_heads'] * HD, cfg['num_key_value_heads'] * HD, cfg['shared_intermediate_size']
if name == 'model.embed_tokens.weight':
return t[:, :E]
if name.startswith('model.switch.'):
return t # codebook and lut, skipped by the converter
if name.endswith('layernorm.weight') or name == 'model.norm.weight':
return t[:E]
if re.search(r'lora_A(_slices\.\d)?$', name):
t = t[:, :, :R, :]
if re.search(r'lora_B(_slices\.\d)?$', name):
t = t[..., :R]
# self attention, q|k|v fused
if name.endswith('qkv_proj.base_layer.weight'):
wq, wk, wv = t.split([q, kv, kv], dim=0)
return torch.cat([wq[:H * HD, :E], wk[:KV * HD, :E], wv[:KV * HD, :E]], dim=0)
if re.search(r'qkv_proj\.lora_A_slices\.\d$', name):
return t[..., :E]
if name.endswith('qkv_proj.lora_B_slices.0'):
return t[:, :, :H * HD, :]
if re.search(r'qkv_proj\.lora_B_slices\.[12]$', name):
return t[:, :, :KV * HD, :]
if name.endswith('o_proj.base_layer.weight'):
return t[:E, :H * HD]
if name.endswith('o_proj.lora_A'):
return t[..., :H * HD]
if name.endswith('o_proj.lora_B'):
return t[:, :, :E, :]
# mlp, gate|up fused
if name.endswith('input_linear.base_layer.weight'):
return t.reshape(2, ff, -1)[:, :FF, :E].reshape(2 * FF, E)
if re.search(r'input_linear\.lora_A_slices\.\d$', name):
return t[..., :E]
if re.search(r'input_linear\.lora_B_slices\.\d$', name):
return t[:, :, :FF, :]
if name.endswith('output_linear.base_layer.weight'):
return t[:E, :FF]
if name.endswith('output_linear.lora_A'):
return t[..., :FF]
if name.endswith('output_linear.lora_B'):
return t[:, :, :E, :]
raise SystemExit(f'unhandled tensor {name}')
def main():
out = Path(sys.argv[1])
out.mkdir(parents=True, exist_ok=True)
r = requests.get(hf_hub_url(REPO, 'config.json'), timeout=120)
r.raise_for_status()
cfg = r.json()
r = requests.get(hf_hub_url(REPO, 'model.safetensors.index.json'), timeout=120)
r.raise_for_status()
weight_map = r.json()['weight_map']
n_weight_layers = max(int(m[1]) for k in weight_map if (m := re.match(r'model\.layers\.(\d+)\.', k))) + 1
jobs = []
for filename in sorted(set(weight_map.values())):
url = hf_hub_url(REPO, filename)
header, base = read_header(url)
jobs += [(url, base, n, m, rename(n)) for n, m in header.items() if rename(n) is not None]
def one(job):
url, base, src, meta, dst = job
return dst, narrow(src, fetch(url, meta, base), cfg).contiguous().clone()
with ThreadPoolExecutor(8) as ex:
tensors = dict(ex.map(one, jobs))
save_file(tensors, out / 'model.safetensors', metadata={'format': 'pt'})
print(f'model.safetensors: {len(tensors)}/{len(weight_map)} tensors, {(out / "model.safetensors").stat().st_size / 1e6:.1f} MB')
for f in AUX:
r = requests.get(hf_hub_url(REPO, f), timeout=120)
r.raise_for_status()
(out / f).parent.mkdir(parents=True, exist_ok=True)
(out / f).write_bytes(r.content)
n_switch = cfg['num_hidden_layers'] - n_weight_layers
n_layers = len(LAYERS) + n_switch
cfg.update(hidden_size=E, intermediate_size=FF, shared_intermediate_size=FF, num_attention_heads=H, num_key_value_heads=KV,
num_hidden_layers=n_layers, layer_types=cfg['layer_types'][:n_layers],
max_lora_rank=R, adapter_ranks=[min(r, R) for r in cfg['adapter_ranks']])
assert cfg['projection_head_dim'] == HD
(out / 'config.json').write_text(json.dumps(cfg, indent=2) + '\n')
main()