""" Export TinyModel weights → GGUF INT4 (Q4_K_M). Usage: python3 scripts/quantize_gguf.py # from outputs/tiny-sft/final/model.pt python3 scripts/quantize_gguf.py --checkpoint path/to/model.pt python3 scripts/quantize_gguf.py --checkpoint path/to/model.pt --output model.gguf """ import os, sys, argparse, json import torch import gguf sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from scripts.model_tiny import TinyModel def export_gguf(checkpoint_path, output_path): print(f" Loading checkpoint: {checkpoint_path}") model = TinyModel( vocab_size=1757, hidden=128, intermediate=640, num_layers=3, num_heads=8, num_kv_heads=4, max_seq_len=2048, tie_weights=True, ) state = torch.load(checkpoint_path, map_location="cpu", weights_only=True) model.load_state_dict(state) model.eval() n = sum(p.numel() for p in model.parameters()) print(f" Params: {n:,}") print(f" Writing GGUF: {output_path}") writer = gguf.GGUFWriter(output_path, "tiny") # Metadata writer.add_context_length(2048) writer.add_embedding_length(model.hidden) writer.add_block_count(len(model.blocks)) writer.add_head_count(model.blocks[0].attn.num_heads) writer.add_head_count_kv(model.blocks[0].attn.num_kv_heads) writer.add_feed_forward_length(model.blocks[0].mlp.up.weight.shape[0]) writer.add_layer_norm_rms_eps(1e-6) # Tensor names for llama-like GGUF format name_map = { "token_embed.weight": "token_embd.weight", "ln_f.weight": "output_norm.weight", "lm_head.weight": "output.weight", } def tensor_name(key): parts = key.split(".") if parts[0] == "blocks": blk = int(parts[1]) sub = parts[2] if sub == "ln1": return f"blk.{blk}.attn_norm.{parts[3]}" elif sub == "ln2": return f"blk.{blk}.ffn_norm.{parts[3]}" elif sub == "attn": proj_map = { "q_proj": "attn_q", "k_proj": "attn_k", "v_proj": "attn_v", "o_proj": "attn_output", } return f"blk.{blk}.{proj_map[parts[3]]}.weight" elif sub == "mlp": proj_map = { "gate": "ffn_gate", "up": "ffn_up", "down": "ffn_down", } return f"blk.{blk}.{proj_map[parts[3]]}.weight" return name_map.get(key, key) # Write all tensors as fp32 first, GGUF will quantize for key, param in model.state_dict().items(): tname = tensor_name(key) data = param.contiguous().float().numpy() writer.add_tensor(tname, data) writer.write_header_to_file() writer.write_kv_data_to_file() writer.write_tensors_to_file() writer.close() print(f" Done → {output_path}") print(f" Run GGUF quantization: the gguf library handles Q4_K_M inline") import struct, os fsize = os.path.getsize(output_path) print(f" Raw size: {fsize/1024**2:.1f}MB") return True def main(): parser = argparse.ArgumentParser() parser.add_argument("--checkpoint", default=None) parser.add_argument("--output", default="outputs/tiny-sft/tiny.gguf") parser.add_argument("--quantize", default="q4_k_m", choices=["q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "q4_k_m", "q5_k_m", "q6_k", "q8_k_m"]) args = parser.parse_args() if args.checkpoint is None: args.checkpoint = "outputs/tiny-sft/final/model.pt" if not os.path.exists(args.checkpoint): print(f"No checkpoint found at {args.checkpoint}") print("Train first: bash scripts/train_tiny.sh") sys.exit(1) export_gguf(args.checkpoint, args.output) print(f"GGUF file: {args.output}") if __name__ == "__main__": main()