#!/usr/bin/env python3 """ Build a tiny slice of ibm-granite/granite-switch-4.1-8b (granite-switch decision model) for testing. Output is gibberish, only the shapes and code paths are real. Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. Besides dropping layers, the hidden sizes are narrowed, because the 100355 x 4096 token embedding alone is 800 MB in BF16. The GGUF budget is 10 MB. Granite Switch embeds N LoRA adapters (here guardian-core and the decision aLoRA) stacked over an adapter dim: lora_A [n_adapters, 1, rank, in], lora_B [n_adapters, 1, out, rank]. Both adapters are kept, the in/out dims are narrowed with the trunk and the rank is cut to its first R components (A rows, B columns). num_hidden_layers counts the weightless switch layer(s) (2 here: 42 = 40 weight layers + 2), the patched config keeps that difference. usage: slice_granite_switch.py """ import json, re, struct, sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path import requests import torch from huggingface_hub import hf_hub_url from safetensors.torch import save_file REPO = 'ibm-granite/granite-switch-4.1-8b' AUX = ['tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.json', 'merges.txt', 'chat_template.jinja', 'generation_config.json', 'adapter_index.json', 'io_configs/decision/io.yaml', 'io_configs/guardian-core/io.yaml'] E, H, KV, FF = 256, 2, 1, 128 # hidden, heads, kv heads, ffn. head_dim stays 128: the granite converter writes no head_dim, llama.cpp derives it as hidden / heads HD = 128 R = 16 # max LoRA rank, 64 -> 16 LAYERS = [0, 1] DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16, 'I64': torch.int64} def read_header(url): def rng(a, b): r = requests.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=120) r.raise_for_status() return r.content n = struct.unpack('