Combined GGUF convert + quantize script (Round 3)

convert-and-quantize-gguf.py

@@ -0,0 +1,317 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "unsloth",
+#     "huggingface_hub>=0.25",
+#     "torch>=2.0",
+#     "safetensors",
+#     "numpy",
+#     "sentencepiece",
+#     "transformers>=4.50",
+#     "gguf>=0.6",
+#     "peft>=0.13",
+#     "cmake",
+# ]
+# ///
+"""
+QR-Verse AI — Combined LoRA → F16 GGUF → Q4_K_M (Single HF Job)
+=================================================================
+
+Merges Round 3 LoRA adapter into base Qwen3-VL-8B, converts to F16 GGUF,
+then quantizes to Q4_K_M. All in one job to avoid double GPU spin-up.
+
+Steps:
+  1. Clone llama.cpp + build llama-quantize (parallel with model download)
+  2. Load base Qwen3-VL-8B + LoRA adapter via Unsloth (FP16)
+  3. Merge LoRA weights into base model
+  4. Save merged model as FP16 safetensors
+  5. Convert to F16 GGUF via convert_hf_to_gguf.py
+  6. Quantize F16 → Q4_K_M via llama-quantize
+  7. Upload Q4_K_M GGUF + Modelfile to HuggingFace Hub
+
+Usage:
+    hf jobs uv run --flavor a100-large --timeout 3h \
+        --secrets HF_TOKEN \
+        https://huggingface.co/Qrverse/qr-verse-ai-lora/resolve/main/convert-and-quantize-gguf.py
+"""
+
+import os
+import sys
+import subprocess
+import logging
+import json
+import threading
+import time
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+ADAPTER_REPO = "Qrverse/qr-verse-ai-lora"
+BASE_MODEL = "unsloth/Qwen3-VL-8B-Instruct"
+F16_FILENAME = "qr-verse-ai-r3-F16.gguf"
+Q4_FILENAME = "qr-verse-ai-r3-Q4_K_M.gguf"
+MERGED_DIR = "./merged-model"
+OUTPUT_DIR = "./output"
+
+SYSTEM_PROMPT = (
+    "You are QR-Verse AI, a helpful assistant for the QR-Verse platform. "
+    "You help users create, customize, and manage QR codes. You can generate "
+    "QR codes for URLs, WiFi networks, vCards, email, SMS, and 20+ other types. "
+    "You also support AI-powered QR code art generation with 130+ style presets. "
+    "You can check website health, SEO, SSL certificates, and broken links. "
+    "You speak 7 languages: English, Spanish, Dutch, French, Portuguese, German, Italian. "
+    "Always be concise, accurate, and helpful."
+)
+
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+start_time = time.time()
+
+
+# ---------------------------------------------------------------------------
+# 1. Clone llama.cpp + build llama-quantize (in background thread)
+# ---------------------------------------------------------------------------
+
+build_error = None
+
+def build_llama_cpp():
+    """Build llama-quantize in background while model loads."""
+    global build_error
+    try:
+        logger.info("[BUILD] Cloning llama.cpp...")
+        subprocess.run(
+            ["git", "clone", "--depth", "1", "https://github.com/ggml-org/llama.cpp.git"],
+            check=True, capture_output=True,
+        )
+
+        logger.info("[BUILD] Building llama-quantize with cmake...")
+        os.makedirs("llama.cpp/build", exist_ok=True)
+        subprocess.run(
+            ["cmake", "-B", "llama.cpp/build", "-S", "llama.cpp",
+             "-DGGML_CUDA=OFF", "-DCMAKE_BUILD_TYPE=Release"],
+            check=True, capture_output=True,
+        )
+        subprocess.run(
+            ["cmake", "--build", "llama.cpp/build", "--target", "llama-quantize", "-j", "4"],
+            check=True, capture_output=True,
+        )
+        logger.info("[BUILD] llama-quantize built successfully")
+    except Exception as e:
+        build_error = e
+        logger.error("[BUILD] Failed: %s", e)
+
+# Start build in background
+build_thread = threading.Thread(target=build_llama_cpp)
+build_thread.start()
+
+
+# ---------------------------------------------------------------------------
+# 2. Load base model + LoRA adapter via Unsloth
+# ---------------------------------------------------------------------------
+
+logger.info("[MODEL] Loading base model: %s (FP16, no quantization)", BASE_MODEL)
+
+from unsloth import FastVisionModel
+
+model, tokenizer = FastVisionModel.from_pretrained(
+    BASE_MODEL,
+    load_in_4bit=False,  # FP16 — clean weights for GGUF
+    max_seq_length=4096,
+)
+
+logger.info("[MODEL] Base model loaded (%.1fs). Applying LoRA adapter: %s",
+            time.time() - start_time, ADAPTER_REPO)
+
+from peft import PeftModel
+model = PeftModel.from_pretrained(model, ADAPTER_REPO)
+logger.info("[MODEL] LoRA adapter applied (%.1fs)", time.time() - start_time)
+
+
+# ---------------------------------------------------------------------------
+# 3. Merge LoRA into base model and save as FP16
+# ---------------------------------------------------------------------------
+
+logger.info("[MERGE] Merging LoRA weights into base model...")
+model = model.merge_and_unload()
+logger.info("[MERGE] Merge complete (%.1fs). Saving to: %s",
+            time.time() - start_time, MERGED_DIR)
+
+model.save_pretrained(MERGED_DIR, safe_serialization=True)
+tokenizer.save_pretrained(MERGED_DIR)
+
+# Free GPU memory — no longer needed
+del model
+del tokenizer
+import torch
+torch.cuda.empty_cache()
+import gc
+gc.collect()
+logger.info("[MERGE] Model saved, GPU memory freed (%.1fs)", time.time() - start_time)
+
+# CRITICAL: Remove quantization_config from config.json
+config_path = os.path.join(MERGED_DIR, "config.json")
+if os.path.exists(config_path):
+    with open(config_path) as f:
+        config = json.load(f)
+    if "quantization_config" in config:
+        logger.info("[MERGE] Removing quantization_config from config.json")
+        del config["quantization_config"]
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+# Copy vision processor configs from adapter repo
+from huggingface_hub import hf_hub_download
+import shutil
+for config_file in ["preprocessor_config.json", "video_preprocessor_config.json", "chat_template.jinja"]:
+    try:
+        src = hf_hub_download(ADAPTER_REPO, config_file)
+        shutil.copy2(src, os.path.join(MERGED_DIR, config_file))
+        logger.info("[MERGE] Copied %s", config_file)
+    except Exception:
+        pass
+
+# Log merged model size
+total_size = 0
+for f in sorted(os.listdir(MERGED_DIR)):
+    fpath = os.path.join(MERGED_DIR, f)
+    if os.path.isfile(fpath):
+        total_size += os.path.getsize(fpath) / 1024 / 1024
+logger.info("[MERGE] Total merged model: %.1f MB", total_size)
+
+
+# ---------------------------------------------------------------------------
+# 4. Convert merged model to F16 GGUF
+# ---------------------------------------------------------------------------
+
+logger.info("[GGUF] Converting merged model to F16 GGUF...")
+
+convert_script = "llama.cpp/convert_hf_to_gguf.py"
+f16_path = os.path.join(OUTPUT_DIR, F16_FILENAME)
+
+result = subprocess.run(
+    [sys.executable, convert_script, MERGED_DIR,
+     "--outfile", f16_path, "--outtype", "f16"],
+    capture_output=True, text=True,
+)
+
+if result.stdout:
+    for line in result.stdout.strip().split("\n")[-10:]:
+        logger.info("  convert: %s", line)
+if result.stderr:
+    for line in result.stderr.strip().split("\n")[-10:]:
+        logger.info("  convert (stderr): %s", line)
+
+if result.returncode != 0:
+    logger.error("[GGUF] F16 conversion failed (exit %d)", result.returncode)
+    logger.error("STDERR: %s", result.stderr[-3000:] if result.stderr else "(empty)")
+    sys.exit(1)
+
+f16_size_gb = os.path.getsize(f16_path) / 1024**3
+logger.info("[GGUF] F16 GGUF created: %s (%.1f GB) (%.1fs)",
+            F16_FILENAME, f16_size_gb, time.time() - start_time)
+
+
+# ---------------------------------------------------------------------------
+# 5. Quantize F16 → Q4_K_M
+# ---------------------------------------------------------------------------
+
+# Wait for llama-quantize build to finish
+build_thread.join()
+if build_error:
+    logger.error("[QUANTIZE] llama-quantize build failed, cannot quantize: %s", build_error)
+    sys.exit(1)
+
+# Find quantize binary
+quantize_bin = "llama.cpp/build/bin/llama-quantize"
+if not os.path.exists(quantize_bin):
+    for candidate in ["llama.cpp/build/llama-quantize", "llama.cpp/build/bin/quantize"]:
+        if os.path.exists(candidate):
+            quantize_bin = candidate
+            break
+
+logger.info("[QUANTIZE] Quantizing F16 → Q4_K_M...")
+q4_path = os.path.join(OUTPUT_DIR, Q4_FILENAME)
+
+result = subprocess.run(
+    [quantize_bin, f16_path, q4_path, "Q4_K_M"],
+    capture_output=True, text=True,
+)
+
+if result.stdout:
+    for line in result.stdout.strip().split("\n")[-10:]:
+        logger.info("  quantize: %s", line)
+
+if result.returncode != 0:
+    logger.error("[QUANTIZE] Q4_K_M quantization failed (exit %d)", result.returncode)
+    logger.error("STDERR: %s", result.stderr[-2000:] if result.stderr else "(empty)")
+    sys.exit(1)
+
+q4_size_gb = os.path.getsize(q4_path) / 1024**3
+logger.info("[QUANTIZE] Q4_K_M created: %s (%.2f GB) (%.1fs)",
+            Q4_FILENAME, q4_size_gb, time.time() - start_time)
+
+# Clean up F16 to free disk space (we only upload Q4_K_M)
+os.remove(f16_path)
+logger.info("[QUANTIZE] F16 GGUF removed to free disk space")
+
+
+# ---------------------------------------------------------------------------
+# 6. Upload Q4_K_M GGUF + Modelfile to Hub
+# ---------------------------------------------------------------------------
+
+from huggingface_hub import HfApi
+api = HfApi()
+
+logger.info("[UPLOAD] Uploading Q4_K_M GGUF to Hub...")
+api.upload_file(
+    path_or_fileobj=q4_path,
+    path_in_repo=Q4_FILENAME,
+    repo_id=ADAPTER_REPO,
+    commit_message=f"Round 3 GGUF Q4_K_M: {Q4_FILENAME} ({q4_size_gb:.1f} GB) — LoRA r64, 3766 examples, loss 0.6704",
+)
+logger.info("[UPLOAD] Q4_K_M GGUF uploaded!")
+
+# Generate Modelfile for Ollama
+modelfile_content = f"""# Ollama Modelfile for QR-Verse AI (Round 3)
+# Usage:
+#   ollama create qr-verse-ai -f Modelfile
+#   ollama run qr-verse-ai
+
+FROM ./{Q4_FILENAME}
+
+SYSTEM \"\"\"{SYSTEM_PROMPT}\"\"\"
+
+PARAMETER temperature 0.7
+PARAMETER num_ctx 4096
+"""
+
+modelfile_path = os.path.join(OUTPUT_DIR, "Modelfile")
+with open(modelfile_path, "w") as f:
+    f.write(modelfile_content)
+
+api.upload_file(
+    path_or_fileobj=modelfile_path,
+    path_in_repo="Modelfile",
+    repo_id=ADAPTER_REPO,
+    commit_message="Ollama Modelfile for QR-Verse AI Round 3 (Q4_K_M)",
+)
+logger.info("[UPLOAD] Modelfile uploaded!")
+
+elapsed = time.time() - start_time
+
+print("\n" + "=" * 60)
+print("GGUF CONVERSION + QUANTIZATION COMPLETE")
+print("=" * 60)
+print(f"  Q4_K_M:    {Q4_FILENAME} ({q4_size_gb:.1f} GB)")
+print(f"  Hub:       https://huggingface.co/{ADAPTER_REPO}")
+print(f"  Time:      {elapsed / 60:.1f} minutes")
+print()
+print("Deploy on Ubuntu RTX 3080:")
+print(f"  1. hf download {ADAPTER_REPO} {Q4_FILENAME} Modelfile")
+print("  2. ollama create qr-verse-ai -f Modelfile")
+print("  3. ollama run qr-verse-ai")
+print("=" * 60)