scripts/export_lora_gguf.py

#!/usr/bin/env python3
"""
Convert the published HF PEFT LoRA adapter to GGUF for llama.cpp.

Enables stacking Well-Tuned + Llama Champion + Off the Grid badges:
  base GGUF + --lora plane-mode-study-coach-lora.gguf

Usage:
  python scripts/export_lora_gguf.py
  python scripts/export_lora_gguf.py --dry-run
  PMS_FINETUNED_MODEL=GuusBouwensNL/plane-mode-nemotron-4b-study-coach python scripts/export_lora_gguf.py
"""

from __future__ import annotations

import argparse
import os
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))

from hf_auth import resolve_hf_token  # noqa: E402

DEFAULT_ADAPTER = os.environ.get(
    "PMS_FINETUNED_MODEL",
    "GuusBouwensNL/plane-mode-nemotron-4b-study-coach",
)
DEFAULT_BASE = os.environ.get(
    "PMS_FT_BASE_MODEL",
    "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
)
DEFAULT_OUT = Path(
    os.environ.get(
        "PMS_LLAMACPP_LORA",
        str(ROOT / "models" / "nemotron" / "plane-mode-study-coach-lora.gguf"),
    )
)
DEFAULT_LLAMA_CPP = Path(os.environ.get("LLAMA_CPP_DIR", ROOT / "vendor" / "llama.cpp"))

# HF config.json for Nemotron 4B dense includes unused MoE keys; llama.cpp then
# tags LoRA as nemotron_h_moe while NVIDIA GGUF base is nemotron_h.
_MOE_KEYS = (
    "num_experts_per_tok",
    "n_routed_experts",
    "n_shared_experts",
    "moe_intermediate_size",
    "moe_shared_expert_intermediate_size",
    "moe_latent_size",
    "moe_shared_expert_overlap",
    "norm_topk_prob",
    "routed_scaling_factor",
    "n_group",
)


def _patch_dense_nemotron_hparams() -> None:
    """Strip spurious MoE keys so LoRA GGUF arch matches dense nemotron_h base."""
    import conversion.base as conv_base

    original = conv_base.ModelBase.load_hparams

    @classmethod
    def load_hparams(cls, dir_model, is_mistral_format):  # type: ignore[no-untyped-def]
        try:
            hparams = original.__func__(cls, dir_model, is_mistral_format)  # type: ignore[attr-defined]
        except AttributeError:
            hparams = original(dir_model, is_mistral_format)
        if hparams.get("num_experts_per_tok"):
            for key in _MOE_KEYS:
                hparams.pop(key, None)
        return hparams

    conv_base.ModelBase.load_hparams = load_hparams  # type: ignore[method-assign]


def _run_convert_lora(
    llama_cpp_dir: Path,
    adapter_dir: Path,
    base_model_id: str,
    outfile: Path,
) -> None:
    sys.path.insert(0, str(llama_cpp_dir))
    _patch_dense_nemotron_hparams()
    argv = [
        "convert_lora_to_gguf.py",
        str(adapter_dir),
        "--base-model-id",
        base_model_id,
        "--trust-remote-code",
        "--outfile",
        str(outfile),
        "--outtype",
        "f16",
    ]
    old_argv = sys.argv
    try:
        sys.argv = argv
        import runpy

        runpy.run_path(str(llama_cpp_dir / "convert_lora_to_gguf.py"), run_name="__main__")
    finally:
        sys.argv = old_argv


def _ensure_llama_cpp(llama_cpp_dir: Path) -> Path:
    convert = llama_cpp_dir / "convert_lora_to_gguf.py"
    if convert.exists():
        return llama_cpp_dir
    llama_cpp_dir.parent.mkdir(parents=True, exist_ok=True)
    print(f"Cloning llama.cpp into {llama_cpp_dir}...")
    subprocess.check_call(
        [
            "git",
            "clone",
            "--depth",
            "1",
            "https://github.com/ggml-org/llama.cpp",
            str(llama_cpp_dir),
        ]
    )
    return llama_cpp_dir


def _download_adapter(adapter_repo: str, dest: Path, token: str | None) -> Path:
    from huggingface_hub import snapshot_download

    dest.mkdir(parents=True, exist_ok=True)
    snapshot_download(
        adapter_repo,
        local_dir=str(dest),
        token=token,
        ignore_patterns=["runs/*", "*.tfevents*"],
    )
    return dest


def export_lora_gguf(
    adapter_repo: str,
    base_model_id: str,
    outfile: Path,
    llama_cpp_dir: Path,
    token: str | None,
    dry_run: bool = False,
) -> Path:
    if dry_run:
        print("=== LoRA → GGUF dry run ===")
        print(f"  adapter:   {adapter_repo}")
        print(f"  base:      {base_model_id}")
        print(f"  output:    {outfile}")
        print(f"  llama.cpp: {llama_cpp_dir}")
        return outfile

    llama_cpp_dir = _ensure_llama_cpp(llama_cpp_dir)
    adapter_dir = outfile.parent / ".hf-lora-cache"
    outfile.parent.mkdir(parents=True, exist_ok=True)

    print(f"Downloading adapter {adapter_repo}...")
    _download_adapter(adapter_repo, adapter_dir, token)

    convert_py = llama_cpp_dir / "convert_lora_to_gguf.py"
    if not convert_py.exists():
        raise FileNotFoundError(f"Missing {convert_py}")
    print("Converting PEFT → GGUF LoRA (dense nemotron_h)...")
    _run_convert_lora(llama_cpp_dir, adapter_dir, base_model_id, outfile)
    print(f"Wrote {outfile} ({outfile.stat().st_size / 1e6:.1f} MB)")
    return outfile


def main() -> int:
    parser = argparse.ArgumentParser(description="Export fine-tuned LoRA to GGUF for llama.cpp")
    parser.add_argument("--adapter", default=DEFAULT_ADAPTER)
    parser.add_argument("--base-model-id", default=DEFAULT_BASE)
    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
    parser.add_argument("--llama-cpp-dir", type=Path, default=DEFAULT_LLAMA_CPP)
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args()

    token = resolve_hf_token()
    export_lora_gguf(
        args.adapter,
        args.base_model_id,
        args.out,
        args.llama_cpp_dir,
        token,
        dry_run=args.dry_run,
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())