v6: GRPO with --resume-adapter to start from SFT checkpoint

training/train_grpo.py

@@ -186,6 +186,7 @@ class TrainConfig:
     format_warmup_lr: float = 5e-5
     save_format_warmup_checkpoint: bool = True
     format_warmup_model_id: Optional[str] = os.environ.get("FORMAT_WARMUP_MODEL_ID")
+    resume_adapter: Optional[str] = os.environ.get("RESUME_ADAPTER")
 
     # LoRA
     lora_r: int = 16
@@ -672,8 +673,24 @@ def train(cfg: TrainConfig) -> None:
             fig.savefig(artifact_dir / "holdout_eval_curve.png", dpi=160)
             plt.close(fig)
 
-    # ----- Format warm-start ---------------------------------------------------
+    # ----- Resume from existing adapter or format warm-start ------------------
     global_step = 0
+    if cfg.resume_adapter:
+        print(f"\n=== resuming from adapter: {cfg.resume_adapter} ===", flush=True)
+        from peft import set_peft_model_state_dict
+        from safetensors.torch import load_file
+        from huggingface_hub import hf_hub_download
+        try:
+            adapter_path = hf_hub_download(
+                cfg.resume_adapter, "adapter_model.safetensors", token=_hf_token()
+            )
+            adapter_weights = load_file(adapter_path)
+            set_peft_model_state_dict(policy, adapter_weights)
+            print(f"[resume] loaded {len(adapter_weights)} tensors from {cfg.resume_adapter}", flush=True)
+        except Exception as e:
+            print(f"[resume] WARNING: could not load adapter: {e}", flush=True)
+        cfg.format_warmup = False
+
     if cfg.format_warmup:
         print("\n=== format warm-start (JSON action behavior) ===", flush=True)
         warmup_metrics = run_format_warmup(
@@ -914,6 +931,7 @@ def _parse_args() -> TrainConfig:
     p.add_argument("--no-save-format-warmup", action="store_true")
     p.add_argument("--format-warmup-model-id", default=None)
     p.add_argument("--sample-temperature", type=float, default=None)
+    p.add_argument("--resume-adapter", default=None)
     args = p.parse_args()
 
     cfg = TrainConfig()
@@ -947,6 +965,8 @@ def _parse_args() -> TrainConfig:
         cfg.format_warmup_model_id = args.format_warmup_model_id
     if args.sample_temperature is not None:
         cfg.sample_temperature = args.sample_temperature
+    if args.resume_adapter:
+        cfg.resume_adapter = args.resume_adapter
     if args.no_push:
         cfg.push_to_hub = False
     if args.no_4bit: