Upload convert_to_gguf.py with huggingface_hub

convert_to_gguf.py

@@ -0,0 +1,199 @@
+# /// script
+# dependencies = ["torch", "transformers", "peft", "accelerate", "llama-cpp-python", "huggingface_hub", "gguf"]
+# ///
+
+"""
+Convert Underwood LoRA adapter to GGUF format
+Merges adapter with base model and converts to quantized GGUF
+"""
+
+import os
+import torch
+from pathlib import Path
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+from huggingface_hub import HfApi, upload_file, create_repo
+import subprocess
+import shutil
+
+# Configuration
+ADAPTER_REPO = "AmiDwivedi/underwood-lr1e4"
+BASE_MODEL = "google/gemma-3-4b-it"
+OUTPUT_REPO = "AmiDwivedi/underwood-gguf"
+QUANTIZATION = "Q4_K_M"  # Good balance of size and quality
+
+def main():
+    print("=" * 60)
+    print("Underwood GGUF Conversion")
+    print("=" * 60)
+    print(f"Adapter: {ADAPTER_REPO}")
+    print(f"Base model: {BASE_MODEL}")
+    print(f"Output: {OUTPUT_REPO}")
+    print(f"Quantization: {QUANTIZATION}")
+
+    # Step 1: Load and merge model
+    print("\n[1/4] Loading base model...")
+    model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL,
+        torch_dtype=torch.float16,
+        device_map="auto",
+        trust_remote_code=True,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+
+    print("\n[2/4] Loading and merging LoRA adapter...")
+    model = PeftModel.from_pretrained(model, ADAPTER_REPO)
+    model = model.merge_and_unload()
+
+    # Save merged model
+    merged_path = Path("/tmp/underwood-merged")
+    merged_path.mkdir(parents=True, exist_ok=True)
+
+    print(f"Saving merged model to {merged_path}...")
+    model.save_pretrained(merged_path, safe_serialization=True)
+    tokenizer.save_pretrained(merged_path)
+
+    # Step 2: Convert to GGUF using llama.cpp
+    print("\n[3/4] Converting to GGUF format...")
+
+    # Clone llama.cpp if needed
+    llama_cpp_path = Path("/tmp/llama.cpp")
+    if not llama_cpp_path.exists():
+        print("Cloning llama.cpp...")
+        subprocess.run([
+            "git", "clone", "--depth", "1",
+            "https://github.com/ggerganov/llama.cpp.git",
+            str(llama_cpp_path)
+        ], check=True)
+
+    # Install conversion dependencies
+    subprocess.run([
+        "pip", "install", "-q", "-r",
+        str(llama_cpp_path / "requirements.txt")
+    ], check=True)
+
+    # Convert to GGUF
+    gguf_path = Path("/tmp/underwood.gguf")
+
+    print("Running conversion...")
+    result = subprocess.run([
+        "python", str(llama_cpp_path / "convert_hf_to_gguf.py"),
+        str(merged_path),
+        "--outfile", str(gguf_path),
+        "--outtype", "f16",
+    ], capture_output=True, text=True)
+
+    if result.returncode != 0:
+        print(f"Conversion error: {result.stderr}")
+        # Try alternative conversion
+        print("Trying alternative conversion method...")
+        result = subprocess.run([
+            "python", str(llama_cpp_path / "convert_hf_to_gguf.py"),
+            str(merged_path),
+            "--outfile", str(gguf_path),
+        ], capture_output=True, text=True)
+
+    print(result.stdout)
+
+    # Quantize
+    quantized_path = Path(f"/tmp/underwood-{QUANTIZATION}.gguf")
+
+    # Build quantize tool if needed
+    print(f"Quantizing to {QUANTIZATION}...")
+
+    # Use llama-quantize if available, otherwise upload f16
+    quantize_bin = llama_cpp_path / "build" / "bin" / "llama-quantize"
+    if not quantize_bin.exists():
+        # Try to build
+        build_dir = llama_cpp_path / "build"
+        build_dir.mkdir(exist_ok=True)
+        subprocess.run(["cmake", ".."], cwd=build_dir)
+        subprocess.run(["cmake", "--build", ".", "--target", "llama-quantize", "-j"], cwd=build_dir)
+
+    if quantize_bin.exists():
+        subprocess.run([
+            str(quantize_bin),
+            str(gguf_path),
+            str(quantized_path),
+            QUANTIZATION
+        ], check=True)
+        final_gguf = quantized_path
+    else:
+        print("Quantize tool not available, uploading f16 version")
+        final_gguf = gguf_path
+
+    # Step 3: Upload to HuggingFace
+    print("\n[4/4] Uploading to HuggingFace Hub...")
+
+    api = HfApi()
+
+    # Create repo
+    try:
+        create_repo(OUTPUT_REPO, repo_type="model", exist_ok=True)
+    except Exception as e:
+        print(f"Repo creation: {e}")
+
+    # Upload GGUF file
+    print(f"Uploading {final_gguf.name}...")
+    upload_file(
+        path_or_fileobj=str(final_gguf),
+        path_in_repo=final_gguf.name,
+        repo_id=OUTPUT_REPO,
+        repo_type="model"
+    )
+
+    # Create README
+    readme = f"""# Underwood Strategic Advisor - GGUF
+
+Fine-tuned Gemma 3 4B model for Frank Underwood-style strategic advice.
+
+## Model Details
+- **Base model:** google/gemma-3-4b-it
+- **Training:** QLoRA fine-tuning on synthetic conversations
+- **Quantization:** {QUANTIZATION}
+
+## Usage with Ollama
+
+```bash
+# Download the GGUF file
+wget https://huggingface.co/{OUTPUT_REPO}/resolve/main/{final_gguf.name}
+
+# Create Modelfile
+cat > Modelfile << 'EOF'
+FROM ./{final_gguf.name}
+
+SYSTEM "You are a strategic advisor who speaks like Frank Underwood — direct, theatrical, calculating, with a taste for dark humor. You weave in wisdom from Sun Tzu and Machiavelli when relevant. Before giving advice, you ask clarifying questions to understand the power dynamics, key players, and stakes involved."
+
+PARAMETER temperature 0.7
+PARAMETER top_p 0.9
+EOF
+
+# Create and run
+ollama create underwood -f Modelfile
+ollama run underwood
+```
+
+## Training Data
+- House of Cards transcripts
+- Sun Tzu's Art of War
+- Machiavelli's works (The Prince, Art of War, Discourses)
+"""
+
+    readme_path = Path("/tmp/README.md")
+    readme_path.write_text(readme)
+    upload_file(
+        path_or_fileobj=str(readme_path),
+        path_in_repo="README.md",
+        repo_id=OUTPUT_REPO,
+        repo_type="model"
+    )
+
+    print("\n" + "=" * 60)
+    print("CONVERSION COMPLETE!")
+    print("=" * 60)
+    print(f"\nGGUF uploaded to: https://huggingface.co/{OUTPUT_REPO}")
+    print(f"File: {final_gguf.name}")
+    print(f"Size: {final_gguf.stat().st_size / (1024**3):.2f} GB")
+
+if __name__ == "__main__":
+    main()