Guard auto-resume against adapter-only checkpoints

train_worker.py

@@ -993,6 +993,35 @@ def latest_checkpoint_in_sibling_runs(output_dir: Path) -> Path | None:
     return checkpoints[-1][2]
 
 
+def checkpoint_resume_compatible(checkpoint_dir: Path) -> tuple[bool, str]:
+    if not checkpoint_dir.exists():
+        return False, "path does not exist"
+    if not checkpoint_dir.is_dir():
+        return False, "path is not a directory"
+
+    full_model_markers = (
+        "model.safetensors",
+        "pytorch_model.bin",
+        "model.safetensors.index.json",
+        "pytorch_model.bin.index.json",
+    )
+    if any((checkpoint_dir / marker).exists() for marker in full_model_markers):
+        return True, ""
+
+    adapter_markers = (
+        "adapter_model.safetensors",
+        "adapter_model.bin",
+        "adapter_config.json",
+    )
+    if any((checkpoint_dir / marker).exists() for marker in adapter_markers):
+        return (
+            False,
+            "adapter-only checkpoint (missing full-model checkpoint files required by Trainer resume)",
+        )
+
+    return False, "missing model checkpoint files"
+
+
 def resolve_resume_checkpoint(value: str | None, output_dir: Path) -> str | None:
     requested = to_text(value).lower()
     if requested in {"", "none", "false", "no"}:
@@ -1000,20 +1029,37 @@ def resolve_resume_checkpoint(value: str | None, output_dir: Path) -> str | None
     if requested in {"auto", "latest"}:
         latest = latest_checkpoint_dir(output_dir)
         if latest is not None:
-            return str(latest)
+            compatible, reason = checkpoint_resume_compatible(latest)
+            if compatible:
+                return str(latest)
+            print(
+                "[train_worker][warn] auto-resume skipped latest checkpoint "
+                f"'{latest}' ({reason})."
+            )
         sibling_latest = latest_checkpoint_in_sibling_runs(output_dir=output_dir)
         if sibling_latest is not None:
+            compatible, reason = checkpoint_resume_compatible(sibling_latest)
+            if compatible:
+                print(
+                    "[train_worker][info] auto-resume fallback selected sibling checkpoint: "
+                    f"{sibling_latest}"
+                )
+                return str(sibling_latest)
             print(
-                "[train_worker][info] auto-resume fallback selected sibling checkpoint: "
-                f"{sibling_latest}"
+                "[train_worker][warn] auto-resume skipped sibling checkpoint "
+                f"'{sibling_latest}' ({reason})."
             )
-            return str(sibling_latest)
         return None
     candidate = Path(to_text(value))
     if not candidate.is_absolute():
         candidate = output_dir / candidate
     if candidate.exists():
-        return str(candidate)
+        compatible, reason = checkpoint_resume_compatible(candidate)
+        if compatible:
+            return str(candidate)
+        raise RuntimeError(
+            f"Requested resume checkpoint is not trainer-resume compatible ({reason}): {candidate}"
+        )
     raise RuntimeError(f"Requested resume checkpoint does not exist: {candidate}")