Instructions to use ggml-org-dev/tiny-lev-for-testing with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use ggml-org-dev/tiny-lev-for-testing with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("ggml-org-dev/tiny-Qwen3.5-4B-for-testing") model = PeftModel.from_pretrained(base_model, "ggml-org-dev/tiny-lev-for-testing") - Notebooks
- Google Colab
- Kaggle
Download slice_qwen35_lora.py from ggml-org-dev/tiny-lev-for-testing: direct link, hf CLI and curl.
- Browser
- Download file 12.1 kB
-
https://huggingface.co/ggml-org-dev/tiny-lev-for-testing/resolve/main/slice_qwen35_lora.py
- Command line
-
hf download hf://ggml-org-dev/tiny-lev-for-testing/slice_qwen35_lora.py
-
curl -L -o slice_qwen35_lora.py https://huggingface.co/ggml-org-dev/tiny-lev-for-testing/resolve/main/slice_qwen35_lora.py
12.1 kB
| #!/usr/bin/env python3 | |
| """ | |
| Build tiny slices of a Qwen3.5 base model and of a LoRA decision adapter trained on it, for testing. Output is gibberish, only the shapes and code paths are real. | |
| Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests (only the needed rows) and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. The stock script only drops layers, which cannot go below the 248320 x hidden token embedding, so this one also narrows the hidden sizes. | |
| The adapter is narrowed with the same indices as the base: LoRA A keeps the input columns of its base weight, LoRA B its output rows (the rank is kept). Only the LoRA tensors of the kept layers are kept. | |
| Widths follow the tinyopenjev Qwen3.5 slice: 4 layers (3 linear attention + 1 full attention), hidden 32, FFN 128, 2 heads / 1 KV head (head_dim 256 kept), linear attention 2 key heads / 6 value heads (head dims 128 kept), the MTP layer is kept (narrowed), vision encoder 2 blocks, 2 heads (head_dim kept), out hidden 32 (the text GGUF budget is 10 MB: a 248320 x 32 matrix is 4.5 MB in Q4_0). | |
| usage: | |
| slice_qwen35_lora.py base <base repo> <revision|main> <output dir> | |
| slice_qwen35_lora.py adapter <adapter repo or local dir> <base repo> <output dir> | |
| (only writes the narrowed adapter_model.safetensors and the narrowed head .pt files, the json files are patched by hand) | |
| """ | |
| import json, struct, sys | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import requests | |
| import torch | |
| from huggingface_hub import hf_hub_download, hf_hub_url | |
| from safetensors import safe_open | |
| from safetensors.torch import save_file | |
| E, FF, NH, NKV = 32, 128, 2, 1 # text: hidden, ffn, heads, kv heads. head_dim stays 256 | |
| LNK, LNV = 2, 6 # linear attention key / value heads, head dims stay 128 | |
| KEEP_LAYERS = [0, 1, 2, 3] # linear, linear, linear, full | |
| VNH = 2 # vision heads, head_dim kept, vision ffn = 2 * hidden | |
| VKEEP = [0, 1] # vision blocks | |
| CHUNK = 32 << 20 | |
| DTYPES = {'BF16': torch.bfloat16, 'F32': torch.float32, 'F16': torch.float16} | |
| SIZES = {'BF16': 2, 'F32': 4, 'F16': 2} | |
| sess = requests.Session() | |
| def rng(url, a, b): | |
| for attempt in range(5): | |
| try: | |
| r = sess.get(url, headers={'Range': f'bytes={a}-{b}'}, timeout=600) | |
| r.raise_for_status() | |
| assert len(r.content) == b - a + 1 | |
| return r.content | |
| except Exception: | |
| if attempt == 4: | |
| raise | |
| def read_header(url): | |
| n = struct.unpack('<Q', rng(url, 0, 7))[0] | |
| h = json.loads(rng(url, 8, 8 + n - 1)) | |
| h.pop('__metadata__', None) | |
| return h, 8 + n | |
| def fetch_rows(pool, url, base, meta, rows): | |
| """fetch the given rows (dim 0) of a tensor, None = all rows""" | |
| shape, dt = meta['shape'], meta['dtype'] | |
| a0 = base + meta['data_offsets'][0] | |
| row_bytes = SIZES[dt] | |
| for s in shape[1:]: | |
| row_bytes *= s | |
| rows = list(range(shape[0])) if rows is None else [int(r) for r in rows] | |
| runs = [] # contiguous runs of rows | |
| for r in rows: | |
| if runs and runs[-1][1] == r: | |
| runs[-1][1] = r + 1 | |
| else: | |
| runs.append([r, r + 1]) | |
| jobs = [] # byte ranges, split in chunks | |
| for s, e in runs: | |
| a, b = a0 + s * row_bytes, a0 + e * row_bytes | |
| while a < b: | |
| jobs.append((a, min(a + CHUNK, b) - 1)) | |
| a += CHUNK | |
| data = b''.join(pool.map(lambda j: rng(url, *j), jobs)) | |
| return torch.frombuffer(bytearray(data), dtype=DTYPES[dt]).reshape([len(rows)] + shape[1:]) | |
| def ar(n, start=0): | |
| return torch.arange(start, start + n) | |
| def text_indices(tc): | |
| """(rows, cols) index of each narrowed text weight, by the module name""" | |
| hd, dk, dv = tc['head_dim'], tc['linear_key_head_dim'], tc['linear_value_head_dim'] | |
| nk = tc['linear_num_key_heads'] | |
| H, I = tc['hidden_size'], tc['intermediate_size'] | |
| hid, ff = ar(E), ar(FF) | |
| q_gated = ar(NH * hd * 2) if tc.get('attn_output_gate') else ar(NH * hd) # [q, gate] per head | |
| lin_qkv = torch.cat([ar(LNK * dk), ar(LNK * dk, nk * dk), ar(LNV * dv, 2 * nk * dk)]) | |
| idx2 = { | |
| 'self_attn.q_proj': (q_gated, hid), 'self_attn.k_proj': (ar(NKV * hd), hid), 'self_attn.v_proj': (ar(NKV * hd), hid), | |
| 'self_attn.o_proj': (hid, ar(NH * hd)), | |
| 'mlp.gate_proj': (ff, hid), 'mlp.up_proj': (ff, hid), 'mlp.down_proj': (hid, ff), | |
| 'linear_attn.in_proj_qkv': (lin_qkv, hid), 'linear_attn.in_proj_z': (ar(LNV * dv), hid), | |
| 'linear_attn.in_proj_a': (ar(LNV), hid), 'linear_attn.in_proj_b': (ar(LNV), hid), | |
| 'linear_attn.out_proj': (hid, ar(LNV * dv)), | |
| } | |
| idx1 = { | |
| 'input_layernorm.weight': hid, 'post_attention_layernorm.weight': hid, | |
| 'linear_attn.A_log': ar(LNV), 'linear_attn.dt_bias': ar(LNV), 'linear_attn.conv1d.weight': lin_qkv, | |
| 'linear_attn.norm.weight': None, 'self_attn.q_norm.weight': None, 'self_attn.k_norm.weight': None, | |
| } | |
| glob = { | |
| 'model.language_model.embed_tokens.weight': (None, hid), 'lm_head.weight': (None, hid), | |
| 'model.language_model.norm.weight': (hid, None), | |
| 'mtp.fc.weight': (hid, torch.cat([hid, ar(E, H)])), # input is [embedding, hidden] | |
| 'mtp.norm.weight': (hid, None), 'mtp.pre_fc_norm_embedding.weight': (hid, None), 'mtp.pre_fc_norm_hidden.weight': (hid, None), | |
| } | |
| del I | |
| return idx2, idx1, glob | |
| def vision_indices(vc): | |
| Vo = vc['hidden_size'] | |
| V = VNH * (Vo // vc['num_heads']) | |
| VFF = 2 * V | |
| m = vc['spatial_merge_size'] ** 2 | |
| v = ar(V) | |
| qkv = torch.cat([ar(V, i * Vo) for i in range(3)]) | |
| blk = { | |
| 'attn.qkv.weight': (qkv, v), 'attn.qkv.bias': (qkv, None), 'attn.proj.weight': (v, v), 'attn.proj.bias': (v, None), | |
| 'mlp.linear_fc1.weight': (ar(VFF), v), 'mlp.linear_fc1.bias': (ar(VFF), None), | |
| 'mlp.linear_fc2.weight': (v, ar(VFF)), 'mlp.linear_fc2.bias': (v, None), | |
| 'norm1.weight': (v, None), 'norm1.bias': (v, None), 'norm2.weight': (v, None), 'norm2.bias': (v, None), | |
| } | |
| glob = { | |
| 'model.visual.patch_embed.proj.weight': (v, None), 'model.visual.patch_embed.proj.bias': (v, None), | |
| 'model.visual.pos_embed.weight': (None, v), | |
| 'model.visual.merger.norm.weight': (v, None), 'model.visual.merger.norm.bias': (v, None), | |
| 'model.visual.merger.linear_fc1.weight': (ar(m * V), torch.cat([ar(V, i * Vo) for i in range(m)])), # m patches are merged | |
| 'model.visual.merger.linear_fc1.bias': (ar(m * V), None), | |
| 'model.visual.merger.linear_fc2.weight': (ar(E), ar(m * V)), 'model.visual.merger.linear_fc2.bias': (ar(E), None), | |
| } | |
| return blk, glob, V, VFF | |
| def plan(name, cfg): | |
| """new name and (rows, cols) for a base tensor, None to drop it""" | |
| tc, vc = cfg['text_config'], cfg['vision_config'] | |
| idx2, idx1, tglob = text_indices(tc) | |
| vblk, vglob, _, _ = vision_indices(vc) | |
| if name in tglob: | |
| return name, tglob[name] | |
| if name in vglob: | |
| return name, vglob[name] | |
| for prefix, keep in (('model.language_model.layers.', KEEP_LAYERS), ('mtp.layers.', None)): | |
| if name.startswith(prefix): | |
| i, key = name[len(prefix):].split('.', 1) | |
| if keep is not None and int(i) not in keep: | |
| return None | |
| new = prefix + str(keep.index(int(i)) if keep is not None else i) + '.' + key | |
| if key in idx1: | |
| return new, (idx1[key], None) | |
| mod = key.removesuffix('.weight') | |
| if mod in idx2: | |
| return new, idx2[mod] | |
| raise SystemExit(f'unhandled tensor {name}') | |
| if name.startswith('model.visual.blocks.'): | |
| i, key = name[len('model.visual.blocks.'):].split('.', 1) | |
| if int(i) not in VKEEP: | |
| return None | |
| return f'model.visual.blocks.{VKEEP.index(int(i))}.{key}', vblk[key] | |
| raise SystemExit(f'unhandled tensor {name}') | |
| def patch_config(c): | |
| tc, vc = c['text_config'], c['vision_config'] | |
| _, _, V, VFF = vision_indices(vc) | |
| tc.update(hidden_size=E, intermediate_size=FF, num_attention_heads=NH, num_key_value_heads=NKV, | |
| linear_num_key_heads=LNK, linear_num_value_heads=LNV, | |
| num_hidden_layers=len(KEEP_LAYERS), layer_types=[tc['layer_types'][i] for i in KEEP_LAYERS]) | |
| vc.update(depth=len(VKEEP), hidden_size=V, intermediate_size=VFF, num_heads=VNH, out_hidden_size=E) | |
| return c | |
| def slice_base(repo, rev, out): | |
| rev = None if rev == 'main' else rev | |
| cfg = json.load(open(hf_hub_download(repo, 'config.json', revision=rev))) | |
| index = json.load(open(hf_hub_download(repo, 'model.safetensors.index.json', revision=rev)))['weight_map'] | |
| tensors = {} | |
| with ThreadPoolExecutor(8) as pool: | |
| for shard in sorted(set(index.values())): | |
| url = hf_hub_url(repo, shard, revision=rev) | |
| header, base = read_header(url) | |
| for name, meta in header.items(): | |
| p = plan(name, cfg) | |
| if p is None: | |
| continue | |
| new, (rows, cols) = p | |
| t = fetch_rows(pool, url, base, meta, rows) | |
| if cols is not None: | |
| t = t[:, cols] | |
| tensors[new] = t.contiguous().clone() | |
| print(f'{name} {meta["shape"]} -> {new} {list(t.shape)}', flush=True) | |
| out.mkdir(parents=True, exist_ok=True) | |
| save_file(dict(sorted(tensors.items())), out / 'model.safetensors', metadata={'format': 'pt'}) | |
| (out / 'config.json').write_text(json.dumps(patch_config(cfg), indent=4) + '\n') | |
| print(f'{len(tensors)} tensors, {(out / "model.safetensors").stat().st_size / 1e6:.1f} MB') | |
| def slice_adapter(src, base_repo, out): | |
| cfg = json.load(open(hf_hub_download(base_repo, 'config.json'))) # revisions of a base share the config | |
| idx2, _, _ = text_indices(cfg['text_config']) | |
| local = Path(src) | |
| if local.is_dir(): | |
| f = safe_open(local / 'adapter_model.safetensors', 'pt') | |
| names, get = list(f.keys()), f.get_tensor | |
| else: | |
| url = hf_hub_url(src, 'adapter_model.safetensors') | |
| header, base = read_header(url) | |
| pool = ThreadPoolExecutor(8) | |
| names, get = list(header), lambda n: fetch_rows(pool, url, base, header[n], None) | |
| tensors = {} | |
| for name in names: | |
| pre, rest = name.split('layers.', 1) | |
| i, key = rest.split('.', 1) | |
| if int(i) not in KEEP_LAYERS: | |
| continue | |
| mod, part = key.split('.lora_') | |
| rows, cols = idx2[mod] | |
| t = get(name) | |
| t = t[:, cols] if part == 'A.weight' else t[rows] | |
| tensors[f'{pre}layers.{KEEP_LAYERS.index(int(i))}.{key}'] = t.contiguous().clone() | |
| print(f'{name} -> {list(t.shape)}', flush=True) | |
| out.mkdir(parents=True, exist_ok=True) | |
| save_file(dict(sorted(tensors.items())), out / 'adapter_model.safetensors', metadata={'format': 'pt'}) | |
| print(f'{len(tensors)} of {len(names)} tensors, {(out / "adapter_model.safetensors").stat().st_size / 1e6:.1f} MB') | |
| # extra heads, they read the hidden state: keep the first E input columns | |
| for fname, keys in (('head.pt', ['q.weight', 'k.weight']), ('mode_b_head.pt', ['question_proj.weight', 'candidate_proj.weight'])): | |
| try: | |
| p = local / fname if local.is_dir() else Path(hf_hub_download(src, fname)) | |
| except Exception: | |
| continue | |
| if not p.is_file(): | |
| continue | |
| h = torch.load(p, map_location='cpu', weights_only=True) | |
| d = h['head'] if 'head' in h else h | |
| for k in keys: | |
| d[k] = d[k][:, :E].contiguous().clone() | |
| torch.save(h, out / fname) | |
| print(f'{fname}: ' + ', '.join(f'{k} {list(d[k].shape)}' for k in keys)) | |
| if __name__ == '__main__': | |
| if sys.argv[1] == 'base': | |
| slice_base(sys.argv[2], sys.argv[3], Path(sys.argv[4])) | |
| elif sys.argv[1] == 'adapter': | |
| slice_adapter(sys.argv[2], sys.argv[3], Path(sys.argv[4])) | |
| else: | |
| raise SystemExit(__doc__) | |