File size: 1,056 Bytes
1f834c5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Merge LoRA adapter and export to GGUF for llama.cpp."""
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel
MODEL_ID = "Qwen/Qwen2.5-1.5B-Instruct"
ADAPTER_DIR = "./adapter-model"
MERGED_DIR = "./merged-model"
print("Loading base model...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID, torch_dtype=torch.float16, device_map="cpu", trust_remote_code=True
)
print("Loading adapter...")
model = PeftModel.from_pretrained(model, ADAPTER_DIR)
print("Merging...")
model = model.merge_and_unload()
print(f"Saving merged model to {MERGED_DIR}...")
model.save_pretrained(MERGED_DIR)
tokenizer.save_pretrained(MERGED_DIR)
print(f"""
Done! Now convert to GGUF:
pip install llama-cpp-python
python -m llama_cpp.convert {MERGED_DIR} --outfile adapter-q8.gguf --outtype q8_0
Or use llama.cpp's convert script:
python /opt/llama.cpp/convert_hf_to_gguf.py {MERGED_DIR} --outfile adapter-q8.gguf --outtype q8_0
""")
|