Upload convert_survival_gguf.py with huggingface_hub

convert_survival_gguf.py

@@ -0,0 +1,78 @@
+
+# /// script
+# dependencies = ["peft", "transformers", "torch", "huggingface_hub", "sentencepiece"]
+# ///
+
+import os
+import subprocess
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from huggingface_hub import HfApi, create_repo
+
+# Configuration
+BASE_MODEL_ID = "meta-llama/Llama-3.2-3B-Instruct"
+ADAPTER_ID = "sunkencity/survival-expert-llama-3b"
+OUTPUT_REPO = "sunkencity/survival-expert-3b-gguf"
+MERGED_DIR = "merged_model"
+GGUF_FILE = "survival-expert-llama-3b.Q4_K_M.gguf"
+
+print(f"Loading base model: {BASE_MODEL_ID}")
+base_model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL_ID,
+    device_map="auto",
+    torch_dtype="auto",
+    trust_remote_code=True
+)
+
+print(f"Loading adapter: {ADAPTER_ID}")
+model = PeftModel.from_pretrained(base_model, ADAPTER_ID)
+
+print("Merging model...")
+model = model.merge_and_unload()
+
+print(f"Saving merged model to {MERGED_DIR}...")
+model.save_pretrained(MERGED_DIR)
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_ID)
+tokenizer.save_pretrained(MERGED_DIR)
+
+print("Cloning llama.cpp...")
+subprocess.run(["git", "clone", "https://github.com/ggerganov/llama.cpp"], check=True)
+
+print("Installing llama.cpp requirements...")
+subprocess.run(["pip", "install", "-r", "llama.cpp/requirements.txt"], check=True)
+
+print("Converting to GGUF (Q4_K_M)...")
+# Note: Newer llama.cpp uses convert_hf_to_gguf.py
+# We perform quantization in two steps: convert to fp16 gguf, then quantize
+# Or if convert script supports outtype...
+
+# Step 1: Convert to FP16 GGUF
+subprocess.run([
+    "python", "llama.cpp/convert_hf_to_gguf.py",
+    MERGED_DIR,
+    "--outfile", "merged_fp16.gguf",
+    "--outtype", "f16"
+], check=True)
+
+# Step 2: Quantize to Q4_K_M
+subprocess.run(["make", "-C", "llama.cpp", "llama-quantize"], check=True)
+subprocess.run([
+    "./llama.cpp/llama-quantize",
+    "merged_fp16.gguf",
+    GGUF_FILE,
+    "Q4_K_M"
+], check=True)
+
+print(f"Creating repo {OUTPUT_REPO}...")
+api = HfApi()
+create_repo(OUTPUT_REPO, repo_type="model", exist_ok=True)
+
+print(f"Uploading {GGUF_FILE}...")
+api.upload_file(
+    path_or_fileobj=GGUF_FILE,
+    path_in_repo=GGUF_FILE,
+    repo_id=OUTPUT_REPO,
+    repo_type="model"
+)
+
+print("Done! GGUF available at:", f"https://huggingface.co/{OUTPUT_REPO}")