#!/usr/bin/env python3 """ Build tiny slices of a Qwen3.5 base model and of a LoRA decision adapter trained on it, for testing. Output is gibberish, only the shapes and code paths are real. Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests (only the needed rows) and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. The stock script only drops layers, which cannot go below the 248320 x hidden token embedding, so this one also narrows the hidden sizes. The adapter is narrowed with the same indices as the base: LoRA A keeps the input columns of its base weight, LoRA B its output rows (the rank is kept). Only the LoRA tensors of the kept layers are kept. Widths follow the tinyopenjev Qwen3.5 slice: 4 layers (3 linear attention + 1 full attention), hidden 32, FFN 128, 2 heads / 1 KV head (head_dim 256 kept), linear attention 2 key heads / 6 value heads (head dims 128 kept), the MTP layer is kept (narrowed), vision encoder 2 blocks, 2 heads (head_dim kept), out hidden 32 (the text GGUF budget is 10 MB: a 248320 x 32 matrix is 4.5 MB in Q4_0). usage: slice_qwen35_lora.py base slice_qwen35_lora.py adapter (only writes the narrowed adapter_model.safetensors and the narrowed head .pt files, the json files are patched by hand) """ import json, struct, sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path import requests import torch from huggingface_hub import hf_hub_download, hf_hub_url from safetensors import safe_open from safetensors.torch import save_file E, FF, NH, NKV = 32, 128, 2, 1 # text: hidden, ffn, heads, kv heads. head_dim stays 256 LNK, LNV = 2, 6 # linear attention key / value heads, head dims stay 128 KEEP_LAYERS = [0, 1, 2, 3] # linear, linear, linear, full VNH = 2 # vision heads, head_dim kept, vision ffn = 2 * hidden VKEEP = [0, 1] # vision blocks CHUNK = 32 << 20 DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16} SIZES = {'BF16': 2, 'F32': 4, 'F16': 2} sess = requests.Session() def rng(url, a, b): for attempt in range(5): try: r = sess.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=600) r.raise_for_status() assert len(r.content) == b - a + 1 return r.content except Exception: if attempt == 4: raise def read_header(url): n = struct.unpack(' {new} {list(t.shape)}', flush=True) out.mkdir(parents=True, exist_ok=True) save_file(dict(sorted(tensors.items())), out / 'model.safetensors', metadata={'format': 'pt'}) (out / 'config.json').write_text(json.dumps(patch_config(cfg), indent=4) + '\n') print(f'{len(tensors)} tensors, {(out / "model.safetensors").stat().st_size / 1e6:.1f} MB') def slice_adapter(src, base_repo, out): cfg = json.load(open(hf_hub_download(base_repo, 'config.json'))) # revisions of a base share the config idx2, _, _ = text_indices(cfg['text_config']) local = Path(src) if local.is_dir(): f = safe_open(local / 'adapter_model.safetensors', 'pt') names, get = list(f.keys()), f.get_tensor else: url = hf_hub_url(src, 'adapter_model.safetensors') header, base = read_header(url) pool = ThreadPoolExecutor(8) names, get = list(header), lambda n: fetch_rows(pool, url, base, header[n], None) tensors = {} for name in names: pre, rest = name.split('layers.', 1) i, key = rest.split('.', 1) if int(i) not in KEEP_LAYERS: continue mod, part = key.split('.lora_') rows, cols = idx2[mod] t = get(name) t = t[:, cols] if part == 'A.weight' else t[rows] tensors[f'{pre}layers.{KEEP_LAYERS.index(int(i))}.{key}'] = t.contiguous().clone() print(f'{name} -> {list(t.shape)}', flush=True) out.mkdir(parents=True, exist_ok=True) save_file(dict(sorted(tensors.items())), out / 'adapter_model.safetensors', metadata={'format': 'pt'}) print(f'{len(tensors)} of {len(names)} tensors, {(out / "adapter_model.safetensors").stat().st_size / 1e6:.1f} MB') # extra heads, they read the hidden state: keep the first E input columns for fname, keys in (('head.pt', ['q.weight', 'k.weight']), ('mode_b_head.pt', ['question_proj.weight', 'candidate_proj.weight'])): try: p = local / fname if local.is_dir() else Path(hf_hub_download(src, fname)) except Exception: continue if not p.is_file(): continue h = torch.load(p, map_location='cpu', weights_only=True) d = h['head'] if 'head' in h else h for k in keys: d[k] = d[k][:, :E].contiguous().clone() torch.save(h, out / fname) print(f'{fname}: ' + ', '.join(f'{k} {list(d[k].shape)}' for k in keys)) if __name__ == '__main__': if sys.argv[1] == 'base': slice_base(sys.argv[2], sys.argv[3], Path(sys.argv[4])) elif sys.argv[1] == 'adapter': slice_adapter(sys.argv[2], sys.argv[3], Path(sys.argv[4])) else: raise SystemExit(__doc__)