File size: 3,565 Bytes
02600fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | """
Export a trained LoRA adapter to llama.cpp-compatible GGUF.
This loads the base Qwen2.5-1.5B-Instruct model with the trained LoRA adapter,
merges the weights, and quantizes to the requested GGUF format(s).
Outputs:
outputs/gguf/grimoire-qwen2.5-1.5b-triage-q4_k_m.gguf
outputs/gguf/grimoire-qwen2.5-1.5b-triage-q3_k_m.gguf (optional)
outputs/gguf/grimoire-qwen2.5-1.5b-triage-q2_k.gguf (optional)
Usage:
python train/export_gguf.py
python train/export_gguf.py --methods q4_k_m q3_k_m
python train/export_gguf.py --lora_dir outputs/lora --base_model Qwen/Qwen2.5-1.5B-Instruct
"""
import argparse
from pathlib import Path
def parse_args():
parser = argparse.ArgumentParser(description="Export fine-tuned LoRA to GGUF")
parser.add_argument("--base_model", default="Qwen/Qwen2.5-1.5B-Instruct", help="Base HF model name/path")
parser.add_argument("--lora_dir", default="outputs/lora", help="Directory with LoRA adapter")
parser.add_argument("--output_dir", default="outputs/gguf", help="Where to write .gguf files")
parser.add_argument(
"--methods",
nargs="+",
default=["q4_k_m"],
help="Quantization methods to produce (e.g. q4_k_m q3_k_m q2_k)",
)
parser.add_argument("--max_seq_length", type=int, default=2048)
parser.add_argument("--merged_dir", default="outputs/merged", help="Optional merged HF model output")
return parser.parse_args()
def main(args):
from unsloth import FastLanguageModel
out_dir = Path(args.output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
# This Unsloth build's from_pretrained() doesn't accept adapter_name_or_path
# (TypeError: Qwen2ForCausalLM.__init__() got an unexpected keyword argument
# 'adapter_name_or_path'). Loading the base model separately and attaching
# via plain peft.PeftModel.from_pretrained *works* for inference, but
# save_pretrained_gguf() doesn't recognize a plain PeftModel as PEFT
# ("Model is not a PEFT model. Saving directly without LoRA merge...") and
# then fails on an unrelated weight-conversion bug trying to save it as if
# it were a full model. Pointing model_name directly at the LoRA directory
# (which has adapter_config.json with base_model_name_or_path set) is
# Unsloth's own documented pattern for this and loads base+adapter as a
# single call, correctly tagged as PEFT.
print(f"Loading base model + LoRA adapter from {args.lora_dir} ...")
model, tokenizer = FastLanguageModel.from_pretrained(
model_name=args.lora_dir,
max_seq_length=args.max_seq_length,
dtype=None,
load_in_4bit=True,
)
# Export GGUF(s)
model_name = "grimoire-qwen2.5-1.5b-triage"
for method in args.methods:
print(f"Exporting GGUF with quantization={method} ...")
model.save_pretrained_gguf(
str(out_dir / model_name),
tokenizer,
quantization_method=method,
)
print("Done. Files:")
for f in sorted(out_dir.glob("*.gguf")):
print(f" {f} ({f.stat().st_size / 1e6:.1f} MB)")
# Save merged HF model (useful for non-GGUF inference / debugging)
if args.merged_dir:
merged_dir = Path(args.merged_dir)
merged_dir.mkdir(parents=True, exist_ok=True)
print(f"Saving merged HF model to {merged_dir}")
merged = model.merge_and_unload()
merged.save_pretrained(merged_dir)
tokenizer.save_pretrained(merged_dir)
if __name__ == "__main__":
args = parse_args()
main(args)
|