fix(train): use MiniCPM-V 4.6 model id and official mmproj

Co-authored-by: Codex <chatgpt-codex-connector[bot]@users.noreply.github.com>

RUNBOOK.md

@@ -30,16 +30,39 @@ python train/synth_reports.py --n 12 --out train/data/preview   # eyeball the im
 ```
 
 ## The fine-tune → offline pipeline (run these in order)
+### Step 0: validate offline with the official base GGUF first
+Before fine-tuning, prove the local backend works with OpenBMB's official base GGUF + mmproj:
+
+```bash
+# Install llama.cpp tooling locally.
+brew install llama.cpp
+
+# Download official base GGUF assets, including the official mmproj.
+mkdir -p models
+huggingface-cli download openbmb/MiniCPM-V-4.6-gguf --local-dir ./models
+
+# Point LOCAL_MODEL_PATH at the downloaded base LLM GGUF, and LOCAL_MMPROJ_PATH at the
+# downloaded mmproj GGUF. Use the exact filenames from ./models.
+export EXTRACTOR_BACKEND=local
+export LOCAL_MODEL_PATH=./models/<official-base-model>.gguf
+export LOCAL_MMPROJ_PATH=./models/<official-mmproj>.gguf
+python app.py
+```
+
+Confirm extraction works fully offline before starting the fine-tune.
+
 ```bash
 # 1) Fine-tune MiniCPM-V on Modal (data generated on the box; needs Modal credits).
-#    First confirm MODEL_TYPE/MODEL_ID in train/modal_finetune.py against `swift sft --help`.
+#    First confirm MODEL_TYPE in train/modal_finetune.py against the MiniCPM-V finetune guide.
 modal run train/modal_finetune.py --n 4000 --epochs 2
 
 # 2) Pull adapters from the Modal volume 'blood-test-adapters', then merge.
-python scripts/merge_lora.py --base openbmb/MiniCPM-V-4_6 \
+python scripts/merge_lora.py --base openbmb/MiniCPM-V-4.6 \
     --adapters ./adapters/minicpmv-lab-lora --out ./merged-minicpmv-lab
 
-# 3) Convert to GGUF + mmproj and quantize Q4_K_M (needs a llama.cpp checkout).
+# 3) Convert only the merged LLM to GGUF and quantize Q4_K_M.
+#    The script downloads/reuses the official openbmb/MiniCPM-V-4.6-gguf mmproj because LoRA
+#    touches the LLM, not the vision encoder.
 LLAMA_CPP=./llama.cpp bash scripts/convert_to_gguf.sh ./merged-minicpmv-lab
 
 # 4) Point the app at the local model and go offline.
@@ -68,7 +91,7 @@ EXTRACTOR_BACKEND=local LOCAL_MODEL_PATH=... LOCAL_MMPROJ_PATH=... \
 
 ## Verify-on-hardware (can't be tested in CI)
 1. `train/modal_finetune.py`: confirm the ms-swift `MODEL_TYPE` for **MiniCPM-V 4.6**.
-2. `scripts/convert_to_gguf.sh`: llama.cpp multimodal script paths/`MINICPMV_VERSION` for 4.6.
+2. `scripts/convert_to_gguf.sh`: llama.cpp `convert_hf_to_gguf.py` path and official mmproj filename for 4.6.
 3. `src/extraction/local_minicpmv.py`: the `LOCAL_CHAT_HANDLER` class for your llama-cpp-python build.
 
 ## Still TODO (next PRs, not this one)

scripts/convert_to_gguf.sh

@@ -1,37 +1,35 @@
 #!/usr/bin/env bash
-# Convert the merged, fine-tuned MiniCPM-V into a quantized GGUF + vision projector (mmproj)
-# for llama.cpp, then quantize to Q4_K_M. Produces the two files the offline backend loads:
+# Convert the merged, fine-tuned MiniCPM-V LLM into a quantized GGUF for llama.cpp, and
+# download the official MiniCPM-V 4.6 mmproj. Produces the two files the offline backend loads:
 #   models/minicpmv-lab.Q4_K_M.gguf   (LOCAL_MODEL_PATH)
 #   models/minicpmv-lab.mmproj.gguf   (LOCAL_MMPROJ_PATH)
 #
-# Prereqs: a merged HF model (scripts/merge_lora.py) and a local llama.cpp checkout.
+# LoRA touches the LLM, not the vision encoder, so the official mmproj remains valid.
 #
-# ⚠️ MiniCPM-V GGUF conversion lives under llama.cpp's multimodal tooling and the exact script
-# names/paths move between releases (older: examples/llava/*, newer: tools/mtmd/*). Check your
-# llama.cpp version and adjust the three SCRIPT paths below. The flow is stable; the paths drift.
+# Prereqs: a merged HF model (scripts/merge_lora.py), a local llama.cpp checkout, and
+# huggingface-cli (`pip install huggingface_hub`).
 set -euo pipefail
 
 MERGED="${1:-./merged-minicpmv-lab}"          # merged HF model dir
 LLAMA="${LLAMA_CPP:-./llama.cpp}"             # path to a llama.cpp checkout
 OUT="${OUT_DIR:-./models}"
-VER="${MINICPMV_VERSION:-3}"                  # MiniCPM-V arch version flag; confirm for 4.6
 mkdir -p "$OUT"
 
-echo "==> 1/4  Split vision encoder + LLM (surgery)"
-python "$LLAMA/examples/llava/minicpmv-surgery.py" -m "$MERGED"
+echo "==> 1/3  Download official MiniCPM-V 4.6 GGUF assets (includes mmproj)"
+huggingface-cli download openbmb/MiniCPM-V-4.6-gguf --local-dir "$OUT"
+MMPROJ="$(find "$OUT" -maxdepth 1 -type f -iname '*mmproj*.gguf' | head -n 1)"
+if [[ -z "${MMPROJ:-}" ]]; then
+  echo "No mmproj GGUF found in $OUT after download" >&2
+  exit 1
+fi
+if [[ "$MMPROJ" != "$OUT/minicpmv-lab.mmproj.gguf" ]]; then
+  cp "$MMPROJ" "$OUT/minicpmv-lab.mmproj.gguf"
+fi
 
-echo "==> 2/4  Build the vision projector (mmproj) GGUF"
-python "$LLAMA/examples/llava/minicpmv-convert-image-encoder-to-gguf.py" \
-  -m "$MERGED" \
-  --minicpmv-projector "$MERGED/minicpmv.projector" \
-  --output-dir "$OUT" \
-  --minicpmv_version "$VER"
-mv "$OUT"/*mmproj*.gguf "$OUT/minicpmv-lab.mmproj.gguf" 2>/dev/null || true
+echo "==> 2/3  Convert the merged LLM to GGUF (f16)"
+python "$LLAMA/convert_hf_to_gguf.py" "$MERGED" --outfile "$OUT/minicpmv-lab.f16.gguf"
 
-echo "==> 3/4  Convert the LLM to GGUF (f16)"
-python "$LLAMA/convert_hf_to_gguf.py" "$MERGED/model" --outfile "$OUT/minicpmv-lab.f16.gguf"
-
-echo "==> 4/4  Quantize to Q4_K_M"
+echo "==> 3/3  Quantize to Q4_K_M"
 "$LLAMA/llama-quantize" "$OUT/minicpmv-lab.f16.gguf" "$OUT/minicpmv-lab.Q4_K_M.gguf" Q4_K_M
 
 echo

scripts/merge_lora.py

@@ -2,7 +2,7 @@
 """Merge the LoRA adapters into the MiniCPM-V base → a standalone HF model for GGUF conversion.
 
     python scripts/merge_lora.py \
-        --base openbmb/MiniCPM-V-4_6 \
+        --base openbmb/MiniCPM-V-4.6 \
         --adapters ./adapters/minicpmv-lab-lora \
         --out ./merged-minicpmv-lab
 """

train/modal_finetune.py

@@ -10,19 +10,19 @@ down and convert to GGUF (see scripts/convert_to_gguf.sh).
 
 Running the fine-tune on Modal also satisfies the Modal prize.
 
-⚠️ VERIFY-ON-FIRST-RUN: the ms-swift `--model_type` for the exact MiniCPM-V 4.6 checkpoint and
-its current dataset-format flags. ms-swift evolves; confirm against `swift sft --help` and the
-MiniCPM-V model card, then pin the value in MODEL_TYPE / MODEL_ID below. The data generation,
-conversion, and plumbing are correct; the trainer invocation is the one thing to confirm.
+⚠️ VERIFY-ON-FIRST-RUN: confirm the exact ms-swift/LLaMA-Factory `--model_type` for MiniCPM-V
+4.6 against the finetune guide at github.com/OpenBMB/MiniCPM-V before running. Trainer CLIs
+evolve; pin the value in MODEL_TYPE below after checking the current guide.
 """
 
 from __future__ import annotations
 
 import modal
 
-# TODO(verify): confirm these against the MiniCPM-V 4.6 model card + `swift sft --help`.
-MODEL_ID = "openbmb/MiniCPM-V-4_6"          # HF id of the base vision model
-MODEL_TYPE = "minicpm-v-v2_6-chat"          # ms-swift model_type; confirm the 4.6 value
+# TODO(verify): confirm the exact ms-swift/LLaMA-Factory model_type for MiniCPM-V 4.6
+# against the finetune guide at github.com/OpenBMB/MiniCPM-V before running.
+MODEL_ID = "openbmb/MiniCPM-V-4.6"          # HF id of the base vision model
+MODEL_TYPE = "minicpm-v-v2_6-chat"          # placeholder until confirmed from the guide
 
 app = modal.App("blood-test-finetune")