finetune: explicit LoRAMerge before HF export + GGUF conversion rails

save_hf_pretrained's claimed adapter auto-merge silently exported base
weights; merge explicitly (LoRAMerge -> strip adapters -> unwrap), assert
non-zero adapter B norms and a changed probe weight before writing.
run_gguf_spark2.sh: containerized convert_hf_to_gguf under the same
memory cap + watchdog. Chain verified end-to-end: merged HF export
hash-differs from base on LoRA targets only; Q4_K_S passes the eval gate.

finetune/nemo/run_gguf_spark2.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Convert the merged Warden HF checkpoint to GGUF — ON SPARK2, same host-safety
+# rails as run_merge_spark2.sh (cgroup cap + MemAvailable watchdog).
+#
+# Step 1 (this script, containerized): HF safetensors -> BF16 GGUF via
+#   llama.cpp's convert_hf_to_gguf.py (lazy tensor streaming, low peak RAM,
+#   but rails stay on out of policy after the 2026-06-12 freeze).
+# Step 2 (separate, cheap): llama-quantize BF16 -> Q4_K_S, mmap-streamed.
+set -uo pipefail
+
+NAME=warden-gguf
+MODELS_DIR=/home/imjonezz/models
+LLAMA_DIR=/home/imjonezz/llama.cpp
+MIN_AVAIL_KB=$((12 * 1024 * 1024))   # 12GB floor for the host
+
+mkdir -p "$MODELS_DIR/gguf"
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+
+docker run -d --name "$NAME" \
+  --memory=100g --memory-swap=100g \
+  -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 \
+  -e PYTHONPATH=/llama.cpp/gguf-py \
+  -v "$MODELS_DIR":/models \
+  -v "$LLAMA_DIR":/llama.cpp \
+  --entrypoint python3 nvcr.io/nvidia/nemo:25.11.nemotron_3_nano \
+  /llama.cpp/convert_hf_to_gguf.py /models/nemotron-3-nano-30b-warden-hf \
+  --outfile /models/gguf/warden-nemotron-30b-bf16.gguf \
+  --outtype bf16 || exit 1
+
+echo "container started; watchdog polling every 5s (kill if MemAvailable < 12GB)"
+while [ -n "$(docker ps -q -f name="$NAME")" ]; do
+  avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo)
+  if [ "$avail_kb" -lt "$MIN_AVAIL_KB" ]; then
+    echo "WATCHDOG TRIPPED: MemAvailable ${avail_kb}kB < 12GB — killing $NAME"
+    docker kill "$NAME"
+    docker logs "$NAME" --tail 30
+    exit 2
+  fi
+  sleep 5
+done
+
+code=$(docker inspect "$NAME" --format '{{.State.ExitCode}}')
+echo "=== container exited with code $code; last log lines: ==="
+docker logs "$NAME" --tail 40 2>&1
+docker rm "$NAME" >/dev/null 2>&1 || true
+exit "$code"

finetune/nemo/run_merge_hf.py

@@ -25,7 +25,8 @@ def main():
     import torch
     from megatron.core import dist_checkpointing
     from megatron.bridge.models.conversion.auto_bridge import AutoBridge
-    from megatron.bridge.peft.lora import LoRA
+    from megatron.bridge.peft.lora import LoRA, LoRAMerge
+    from megatron.bridge.peft.lora_layers import LoRALinear
     from megatron.bridge.training.checkpointing import (
         _generate_model_state_dict,
         apply_peft_adapter_filter_to_state_dict,
@@ -37,6 +38,11 @@ def main():
     base_dir = os.environ.get("MEGATRON_CKPT", "/models/nemotron-3-nano-30b-megatron")
     out_dir = os.environ.get("OUT_DIR", "/models/nemotron-3-nano-30b-warden-hf")
 
+    # clear any previous (possibly bad) export — the container runs as root
+    import shutil
+
+    shutil.rmtree(out_dir, ignore_errors=True)
+
     print(f"[merge-hf] base={base_dir} lora={lora_dir} -> {out_dir}", flush=True)
     bridge = AutoBridge.from_hf_pretrained(hf_dir, trust_remote_code=True)
 
@@ -64,15 +70,53 @@ def main():
     sharded_sd = apply_peft_adapter_filter_to_state_dict(sharded_sd, lora_peft)
     loaded = dist_checkpointing.load(sharded_sd, str(lora_dir))
     key = "model" if "model" in loaded else next(k for k in loaded if k.startswith("model"))
-    missing = model[0].load_state_dict(loaded[key], strict=False)
+    # MegatronModule.load_state_dict returns None, not torch's NamedTuple
+    ret = model[0].load_state_dict(loaded[key], strict=False)
+    unexpected = len(ret.unexpected_keys) if ret is not None else "n/a"
     n_adapter = len(loaded[key])
-    print(f"[merge-hf] loaded {n_adapter} adapter tensors (unexpected: {len(missing.unexpected_keys)})", flush=True)
+    print(f"[merge-hf] loaded {n_adapter} adapter tensors (unexpected: {unexpected})", flush=True)
     if n_adapter == 0:
         raise RuntimeError("no adapter tensors loaded — refusing to export an unmodified base model")
 
-    # save_hf_pretrained merges LoRALinear wrappers into dense weights during
-    # its streaming export; source_path preserves the custom Nemotron-H
-    # modeling files so the result is from_pretrained-loadable.
+    # Lesson from the first export: save_hf_pretrained alone wrote a model
+    # byte-identical to the base. Merge explicitly (as the stock merge_lora.py
+    # does) and prove the adapters are non-trivial before exporting.
+    b_norm = sum(
+        m.adapter.linear_out.weight.float().norm().item()
+        for m in model[0].modules()
+        if isinstance(m, LoRALinear)
+    )
+    print(f"[merge-hf] sum of adapter B-matrix norms: {b_norm:.4f}", flush=True)
+    if b_norm == 0.0:
+        raise RuntimeError("all adapter B matrices are zero — checkpoint load did not take")
+
+    probe = max(
+        (m for m in model[0].modules() if isinstance(m, LoRALinear)),
+        key=lambda m: m.adapter.linear_out.weight.float().norm().item(),
+    )
+    before = probe.to_wrap.weight.detach().clone()
+
+    merged = LoRAMerge()(model[0], training=False)
+    for m in merged.modules():
+        if hasattr(m, "adapter"):
+            delattr(m, "adapter")
+
+    def _unwrap(module):
+        for name, child in list(module.named_children()):
+            if isinstance(child, LoRALinear):
+                setattr(module, name, child.to_wrap)
+            else:
+                _unwrap(child)
+
+    _unwrap(merged)
+    model[0] = merged
+
+    if torch.equal(before, probe.to_wrap.weight):
+        raise RuntimeError("probe weight unchanged by LoRAMerge — refusing to export")
+    print("[merge-hf] LoRAMerge applied; probe weight changed as expected", flush=True)
+
+    # source_path preserves the custom Nemotron-H modeling files so the
+    # result is from_pretrained-loadable.
     bridge.save_hf_pretrained(model, out_dir, source_path=hf_dir)
     print("[merge-hf] export complete", flush=True)