feat(train): Modal MiniCPM-V LoRA fine-tune + GGUF conversion scripts

Co-authored-by: Codex <chatgpt-codex-connector[bot]@users.noreply.github.com>

scripts/convert_to_gguf.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Convert the merged, fine-tuned MiniCPM-V into a quantized GGUF + vision projector (mmproj)
+# for llama.cpp, then quantize to Q4_K_M. Produces the two files the offline backend loads:
+#   models/minicpmv-lab.Q4_K_M.gguf   (LOCAL_MODEL_PATH)
+#   models/minicpmv-lab.mmproj.gguf   (LOCAL_MMPROJ_PATH)
+#
+# Prereqs: a merged HF model (scripts/merge_lora.py) and a local llama.cpp checkout.
+#
+# ⚠️ MiniCPM-V GGUF conversion lives under llama.cpp's multimodal tooling and the exact script
+# names/paths move between releases (older: examples/llava/*, newer: tools/mtmd/*). Check your
+# llama.cpp version and adjust the three SCRIPT paths below. The flow is stable; the paths drift.
+set -euo pipefail
+
+MERGED="${1:-./merged-minicpmv-lab}"          # merged HF model dir
+LLAMA="${LLAMA_CPP:-./llama.cpp}"             # path to a llama.cpp checkout
+OUT="${OUT_DIR:-./models}"
+VER="${MINICPMV_VERSION:-3}"                  # MiniCPM-V arch version flag; confirm for 4.6
+mkdir -p "$OUT"
+
+echo "==> 1/4  Split vision encoder + LLM (surgery)"
+python "$LLAMA/examples/llava/minicpmv-surgery.py" -m "$MERGED"
+
+echo "==> 2/4  Build the vision projector (mmproj) GGUF"
+python "$LLAMA/examples/llava/minicpmv-convert-image-encoder-to-gguf.py" \
+  -m "$MERGED" \
+  --minicpmv-projector "$MERGED/minicpmv.projector" \
+  --output-dir "$OUT" \
+  --minicpmv_version "$VER"
+mv "$OUT"/*mmproj*.gguf "$OUT/minicpmv-lab.mmproj.gguf" 2>/dev/null || true
+
+echo "==> 3/4  Convert the LLM to GGUF (f16)"
+python "$LLAMA/convert_hf_to_gguf.py" "$MERGED/model" --outfile "$OUT/minicpmv-lab.f16.gguf"
+
+echo "==> 4/4  Quantize to Q4_K_M"
+"$LLAMA/llama-quantize" "$OUT/minicpmv-lab.f16.gguf" "$OUT/minicpmv-lab.Q4_K_M.gguf" Q4_K_M
+
+echo
+echo "Done. Set in the Space:"
+echo "  LOCAL_MODEL_PATH=$OUT/minicpmv-lab.Q4_K_M.gguf"
+echo "  LOCAL_MMPROJ_PATH=$OUT/minicpmv-lab.mmproj.gguf"
+echo "  EXTRACTOR_BACKEND=local"
+echo "Track both .gguf files with git-lfs and commit them into the Space repo."

scripts/merge_lora.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Merge the LoRA adapters into the MiniCPM-V base → a standalone HF model for GGUF conversion.
+
+    python scripts/merge_lora.py \
+        --base openbmb/MiniCPM-V-4_6 \
+        --adapters ./adapters/minicpmv-lab-lora \
+        --out ./merged-minicpmv-lab
+"""
+
+from __future__ import annotations
+
+import argparse
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base", required=True, help="base model HF id or path")
+    ap.add_argument("--adapters", required=True, help="LoRA adapter dir (from Modal volume)")
+    ap.add_argument("--out", required=True, help="output dir for the merged model")
+    args = ap.parse_args()
+
+    import torch
+    from peft import PeftModel
+    from transformers import AutoModel, AutoProcessor, AutoTokenizer
+
+    print(f"Loading base {args.base} ...")
+    model = AutoModel.from_pretrained(
+        args.base, trust_remote_code=True, torch_dtype=torch.float16
+    )
+    print(f"Applying adapters {args.adapters} ...")
+    model = PeftModel.from_pretrained(model, args.adapters)
+    model = model.merge_and_unload()
+
+    model.save_pretrained(args.out, safe_serialization=True)
+    AutoTokenizer.from_pretrained(args.base, trust_remote_code=True).save_pretrained(args.out)
+    try:
+        AutoProcessor.from_pretrained(args.base, trust_remote_code=True).save_pretrained(args.out)
+    except Exception:
+        pass  # some MiniCPM-V revisions bundle the processor in the tokenizer
+
+    print(f"Merged model written to {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

train/modal_finetune.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""LoRA fine-tune of MiniCPM-V for lab extraction, on Modal.
+
+Strategy: generate the synthetic dataset **on the GPU box** (it's pure Python + PIL, fully
+reproducible from a seed), convert to the vision-SFT format, then LoRA fine-tune MiniCPM-V with
+ms-swift. No image upload, no dataset drift. Adapters are saved to a Modal Volume; pull them
+down and convert to GGUF (see scripts/convert_to_gguf.sh).
+
+    modal run train/modal_finetune.py --n 4000
+
+Running the fine-tune on Modal also satisfies the Modal prize.
+
+⚠️ VERIFY-ON-FIRST-RUN: the ms-swift `--model_type` for the exact MiniCPM-V 4.6 checkpoint and
+its current dataset-format flags. ms-swift evolves; confirm against `swift sft --help` and the
+MiniCPM-V model card, then pin the value in MODEL_TYPE / MODEL_ID below. The data generation,
+conversion, and plumbing are correct; the trainer invocation is the one thing to confirm.
+"""
+
+from __future__ import annotations
+
+import modal
+
+# TODO(verify): confirm these against the MiniCPM-V 4.6 model card + `swift sft --help`.
+MODEL_ID = "openbmb/MiniCPM-V-4_6"          # HF id of the base vision model
+MODEL_TYPE = "minicpm-v-v2_6-chat"          # ms-swift model_type; confirm the 4.6 value
+
+app = modal.App("blood-test-finetune")
+
+image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .apt_install("git")
+    .pip_install(
+        "torch",
+        "transformers>=4.44",
+        "peft>=0.12",
+        "accelerate>=0.33",
+        "pillow>=10",
+        "ms-swift>=2.5",
+        "timm",
+        "sentencepiece",
+    )
+    # Mount our generator + converter + marker reference so the box builds its own data.
+    .add_local_dir("src", "/root/app/src")
+    .add_local_dir("train", "/root/app/train")
+)
+
+adapters = modal.Volume.from_name("blood-test-adapters", create_if_missing=True)
+hf_cache = modal.Volume.from_name("blood-test-hf-cache", create_if_missing=True)
+
+
+@app.function(
+    image=image,
+    gpu="A100",
+    timeout=3 * 60 * 60,
+    volumes={"/adapters": adapters, "/root/.cache/huggingface": hf_cache},
+)
+def train(n: int = 4000, epochs: int = 2, seed: int = 13) -> str:
+    import subprocess
+    import sys
+    from pathlib import Path
+
+    sys.path.insert(0, "/root/app")
+    from train.synth_reports import generate
+    from train.to_sft_dataset import convert
+
+    # 1) build the dataset on the box
+    data_dir = Path("/root/app/train/data/synth")
+    labels = generate(n, data_dir, seed=seed)
+    sft_path = Path("/root/app/train/data/sft.jsonl")
+    n_examples = convert(labels, sft_path)
+    print(f"Generated {n_examples} SFT examples at {sft_path}")
+
+    # 2) LoRA fine-tune with ms-swift
+    out_dir = "/adapters/minicpmv-lab-lora"
+    cmd = [
+        "swift", "sft",
+        "--model_type", MODEL_TYPE,
+        "--model_id_or_path", MODEL_ID,
+        "--sft_type", "lora",
+        "--dataset", str(sft_path),
+        "--num_train_epochs", str(epochs),
+        "--lora_rank", "16",
+        "--lora_alpha", "32",
+        "--learning_rate", "1e-4",
+        "--batch_size", "2",
+        "--gradient_accumulation_steps", "8",
+        "--max_length", "2048",
+        "--output_dir", out_dir,
+        "--save_total_limit", "1",
+    ]
+    print("Running:", " ".join(cmd))
+    subprocess.run(cmd, check=True)
+
+    adapters.commit()
+    return out_dir
+
+
+@app.local_entrypoint()
+def main(n: int = 4000, epochs: int = 2) -> None:
+    path = train.remote(n=n, epochs=epochs)
+    print(f"\nLoRA adapters saved to Modal volume 'blood-test-adapters' at {path}")
+    print("Next: download adapters, merge into the base model, convert to GGUF + mmproj,")
+    print("quantize Q4_K_M, and bundle into the Space (see scripts/convert_to_gguf.sh).")