training: push adapters to HF Hub after SFT + each GRPO stage

HF Spaces /data is ephemeral and silently got wiped mid-run during
testing, killing the SFT adapter and the stage1 GRPO checkpoint.

Push each adapter to a dedicated HF Hub model repo as soon as it's
saved:
shivam2k3/opensoc-defender-grpo-sft (after SFT)
shivam2k3/opensoc-defender-grpo-stage1_basic (after stage1)
shivam2k3/opensoc-defender-grpo-stage2_multi (after stage2)
...
shivam2k3/opensoc-defender-grpo (final, written by
scripts/run_full_pipeline.sh)

A later restart can re-run on a fresh container by downloading the
latest bookmark instead of redoing 12 min of SFT or 16 min per GRPO
stage.

Made-with: Cursor

scripts/run_full_pipeline.sh

@@ -46,6 +46,24 @@ python -m train.sft_warmstart \
   --epochs 1 --batch-size 4 --grad-accum 4 --lr 2e-4 \
   --out checkpoints/defender_sft_adapter
 
+# Cheap insurance: HF Spaces may wipe /data mid-run. Push the SFT
+# adapter to a dedicated HF Hub model repo immediately so a restart
+# can skip step 3 by re-downloading it.
+if [ -n "${HF_TOKEN:-}" ] && [ -n "${HF_ADAPTER_REPO:-}" ]; then
+  echo "    pushing SFT adapter to ${HF_ADAPTER_REPO}-sft for resilience ..."
+  python -c "
+import os
+from huggingface_hub import HfApi, upload_folder
+token = os.environ['HF_TOKEN']
+sft_repo = os.environ['HF_ADAPTER_REPO'] + '-sft'
+api = HfApi(token=token)
+api.create_repo(sft_repo, exist_ok=True, private=False)
+upload_folder(repo_id=sft_repo, folder_path='checkpoints/defender_sft_adapter',
+              commit_message='SFT warm-start checkpoint (resume bookmark)', token=token)
+print('  SFT bookmark ->', sft_repo)
+"
+fi
+
 echo "[4/6] GRPO curriculum (~3 hr on L4) ..."
 python -m train.train_grpo \
   --sft-adapter checkpoints/defender_sft_adapter \

train/train_grpo.py

@@ -182,8 +182,31 @@ def main() -> None:
             callbacks=[json_logger],
         )
         trainer.train()
-        model.save_pretrained(os.path.join(out_dir, stage_id, "adapter"))
-        print(f"Saved {stage_id} adapter to {out_dir}/{stage_id}/adapter")
+        adapter_path = os.path.join(out_dir, stage_id, "adapter")
+        model.save_pretrained(adapter_path)
+        print(f"Saved {stage_id} adapter to {adapter_path}")
+
+        # Resilience: HF Spaces /data is ephemeral, so push each stage
+        # adapter to a stage-specific HF Hub repo path immediately. A
+        # later /data wipe can then resume from the latest stage by
+        # downloading from the Hub.
+        token = os.environ.get("HF_TOKEN")
+        adapter_repo = os.environ.get("HF_ADAPTER_REPO")
+        if token and adapter_repo:
+            try:
+                from huggingface_hub import HfApi, upload_folder
+                staged_repo = f"{adapter_repo}-{stage_id}"
+                api = HfApi(token=token)
+                api.create_repo(staged_repo, exist_ok=True, private=False)
+                upload_folder(
+                    repo_id=staged_repo,
+                    folder_path=adapter_path,
+                    commit_message=f"GRPO {stage_id} adapter checkpoint",
+                    token=token,
+                )
+                print(f"  bookmark -> https://huggingface.co/{staged_repo}")
+            except Exception as e:
+                print(f"  (per-stage push failed: {e}); continuing")
 
     print(f"GRPO curriculum complete. Final adapter: {out_dir}/{stages[-1]}/adapter")