File size: 3,961 Bytes
811a1b2 f0e7567 811a1b2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 |
# /// script
# dependencies = ["transformers", "peft", "huggingface_hub", "torch", "sentencepiece", "protobuf"]
# ///
"""
Convert fine-tuned LoRA model to GGUF format with Q4_K_M quantization.
"""
import os
import subprocess
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
import torch
# Hardcoded configuration
ADAPTER_MODEL = "nathens/qwen-codeforces-sft"
BASE_MODEL = "Qwen/Qwen2.5-0.5B"
OUTPUT_REPO = "nathens/my-model-gguf"
QUANTIZATION = "Q4_K_M"
print(f"🔧 Converting model to GGUF")
print(f" Base model: {BASE_MODEL}")
print(f" Adapter: {ADAPTER_MODEL}")
print(f" Output: {OUTPUT_REPO}")
print(f" Quantization: {QUANTIZATION}")
# Step 1: Load base model and tokenizer
print("\n📦 Loading base model and tokenizer...")
base_model = AutoModelForCausalLM.from_pretrained(
BASE_MODEL,
dtype=torch.float16,
device_map="auto",
trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
# Step 2: Load and merge LoRA adapter
print(f"🔀 Loading and merging LoRA adapter from {ADAPTER_MODEL}...")
model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL)
print("⚙️ Merging adapter weights into base model...")
merged_model = model.merge_and_unload()
# Step 3: Save merged model
print("💾 Saving merged model...")
merged_dir = "./merged_model"
merged_model.save_pretrained(merged_dir)
tokenizer.save_pretrained(merged_dir)
print(f"✅ Merged model saved to {merged_dir}")
# Step 4: Install llama.cpp for conversion
print("\n📥 Installing llama.cpp for GGUF conversion...")
subprocess.run(["apt-get", "update", "-qq"], check=True)
subprocess.run(["apt-get", "install", "-y", "-qq", "git", "build-essential", "cmake"], check=True)
subprocess.run(["git", "clone", "https://github.com/ggerganov/llama.cpp.git"], check=True)
# Build llama.cpp with CMake
nproc_result = subprocess.run(["nproc"], capture_output=True, text=True, check=True)
nproc = nproc_result.stdout.strip()
print(f"Building llama.cpp with {nproc} cores using CMake...")
os.makedirs("llama.cpp/build", exist_ok=True)
subprocess.run(["cmake", "-B", "llama.cpp/build", "-S", "llama.cpp"], check=True)
subprocess.run(["cmake", "--build", "llama.cpp/build", "--config", "Release", "-j", nproc], check=True)
# Step 5: Convert to GGUF format
print("\n🔄 Converting to GGUF format...")
subprocess.run([
"python3", "llama.cpp/convert_hf_to_gguf.py",
merged_dir,
"--outfile", "./model-f16.gguf",
"--outtype", "f16"
], check=True)
print("✅ Converted to FP16 GGUF")
# Step 6: Quantize to Q4_K_M
print(f"\n⚡ Quantizing to {QUANTIZATION}...")
subprocess.run([
"./llama.cpp/build/bin/llama-quantize",
"./model-f16.gguf",
f"./model-{QUANTIZATION}.gguf",
QUANTIZATION
], check=True)
print(f"✅ Quantized to {QUANTIZATION}")
# Step 7: Upload to Hub
print(f"\n📤 Uploading to {OUTPUT_REPO}...")
from huggingface_hub import HfApi
api = HfApi()
# Create repo
try:
api.create_repo(OUTPUT_REPO, repo_type="model", exist_ok=True)
except Exception as e:
print(f"Note: {e}")
# Upload GGUF files
api.upload_file(
path_or_fileobj=f"./model-{QUANTIZATION}.gguf",
path_in_repo=f"model-{QUANTIZATION}.gguf",
repo_id=OUTPUT_REPO,
repo_type="model"
)
api.upload_file(
path_or_fileobj="./model-f16.gguf",
path_in_repo="model-f16.gguf",
repo_id=OUTPUT_REPO,
repo_type="model"
)
# Upload tokenizer files
for file in ["tokenizer.json", "tokenizer_config.json"]:
try:
api.upload_file(
path_or_fileobj=f"{merged_dir}/{file}",
path_in_repo=file,
repo_id=OUTPUT_REPO,
repo_type="model"
)
except Exception:
pass
print(f"\n✅ Conversion complete!")
print(f"📁 GGUF model available at: https://huggingface.co/{OUTPUT_REPO}")
print(f"\n💡 To use with Ollama:")
print(f" huggingface-cli download {OUTPUT_REPO} model-{QUANTIZATION}.gguf")
|