feat(train): live tqdm+ETA, per-update HF Hub checkpoint upload, HF Jobs entrypoint + launcher

colab/train_lib.py

@@ -514,11 +514,65 @@ class PPOTrainer:
         return {k: float(sum(v) / max(len(v), 1)) for k, v in stats.items()}
 
 
+def _fmt_dur(seconds: float) -> str:
+    seconds = int(max(0, seconds))
+    h, rem = divmod(seconds, 3600)
+    m, s   = divmod(rem, 60)
+    if h:  return f"{h}h{m:02d}m{s:02d}s"
+    if m:  return f"{m}m{s:02d}s"
+    return f"{s}s"
+
+
+def _make_hf_pusher(cfg: dict, run_name: str):
+    """Return a callable(local_dir, path_in_repo) that uploads a folder to a
+    private HF model repo. No-op if HF push isn't configured."""
+    push_user = os.environ.get("IC_PUSH_USER", "").strip()
+    if not push_user or not os.environ.get("HF_TOKEN"):
+        print("[ckpt-push] disabled (set IC_PUSH_USER + HF_TOKEN to enable).")
+        return lambda *_a, **_kw: None
+
+    try:
+        from huggingface_hub import HfApi, create_repo
+    except ImportError:                                         # pragma: no cover
+        print("[ckpt-push] huggingface_hub missing — skipping push.")
+        return lambda *_a, **_kw: None
+
+    repo  = f"{push_user}/incident-commander-actor"
+    token = os.environ["HF_TOKEN"]
+    create_repo(repo, exist_ok=True, repo_type="model", token=token,
+                private=False)
+    api = HfApi(token=token)
+    print(f"[ckpt-push] enabled → https://huggingface.co/{repo}")
+
+    def _push(local_dir: Path, path_in_repo: str) -> None:
+        try:
+            api.upload_folder(
+                folder_path=str(local_dir),
+                repo_id=repo,
+                repo_type="model",
+                path_in_repo=path_in_repo,
+                commit_message=f"{run_name}: {path_in_repo}",
+                run_as_future=False,
+            )
+            print(f"[ckpt-push] uploaded {path_in_repo}")
+        except Exception as exc:                                # noqa: BLE001
+            print(f"[ckpt-push] upload failed for {path_in_repo}: {exc}",
+                  file=sys.stderr)
+
+    return _push
+
+
 # ---------------------------------------------------------------------------
 # Public entry point.
 # ---------------------------------------------------------------------------
 def train_loop(cfg: dict | None = None) -> Path:
-    """Run the full training loop. Returns the path to the JSON log."""
+    """Run the full training loop. Returns the path to the JSON log.
+
+    Progress + ETA are printed every update. If `IC_PUSH_USER` and `HF_TOKEN`
+    are set, every checkpoint **and** the live JSON log are streamed to
+    `<user>/incident-commander-actor` on HF Hub — so the moment your compute
+    credits run out, the latest checkpoint is already safe in the cloud.
+    """
     cfg = {**CFG, **(cfg or {})}
     run_name = cfg["run_name"] or f"run_{int(time.time())}"
     log_path = LOGS_DIR / f"training_{run_name}.json"
@@ -526,6 +580,8 @@ def train_loop(cfg: dict | None = None) -> Path:
     print(f"[train] device CUDA?: {torch.cuda.is_available()}  "
           f"name: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else '-'}")
 
+    push_ckpt = _make_hf_pusher(cfg, run_name)
+
     actor  = QwenActor(model_name=cfg["actor_model"],
                        max_seq_len=cfg["max_seq_len"],
                        lora_r=cfg["lora_r"], lora_alpha=cfg["lora_alpha"],
@@ -545,7 +601,23 @@ def train_loop(cfg: dict | None = None) -> Path:
     log: dict[str, Any] = {"config": cfg, "updates": []}
     log_path.write_text(json.dumps(log, indent=2))
 
-    for upd in range(1, cfg["total_updates"] + 1):
+    # tqdm if available, otherwise a no-op shim.
+    try:
+        from tqdm.auto import tqdm
+        bar = tqdm(total=cfg["total_updates"], desc=f"PPO[{run_name}]",
+                   unit="upd", dynamic_ncols=True)
+    except Exception:                                            # noqa: BLE001
+        class _Shim:
+            def update(self, *_a, **_kw): pass
+            def set_postfix_str(self, *_a, **_kw): pass
+            def close(self): pass
+        bar = _Shim()
+
+    total = cfg["total_updates"]
+    train_t0 = time.time()
+    rolling: list[float] = []                                    # last-N step times
+
+    for upd in range(1, total + 1):
         t0 = time.time()
         trans   = collector.collect(cfg["rollouts_per_update"])
         adv_ret = compute_gae(trans, cfg["gamma"], cfg["gae_lambda"])
@@ -556,10 +628,20 @@ def train_loop(cfg: dict | None = None) -> Path:
         for tr in trans:
             ep_rewards.setdefault(tr.task_id, []).append(tr.reward)
         per_ep = {tid: round(sum(rs), 3) for tid, rs in ep_rewards.items()}
-        elapsed = round(time.time() - t0, 2)
+        elapsed = time.time() - t0
+
+        rolling.append(elapsed)
+        rolling = rolling[-10:]                                  # last 10 updates
+        avg_per_upd = sum(rolling) / len(rolling)
+        remaining   = total - upd
+        eta_s       = remaining * avg_per_upd
+        wall        = time.time() - train_t0
+
         entry = {
             "update":          upd,
-            "elapsed_s":       elapsed,
+            "elapsed_s":       round(elapsed, 2),
+            "wall_s":          round(wall, 1),
+            "eta_s":           round(eta_s, 1),
             "n_transitions":   len(trans),
             "mean_reward":     round(sum(t.reward for t in trans) / max(len(trans), 1), 4),
             "mean_value":      round(sum(t.value  for t in trans) / max(len(trans), 1), 4),
@@ -568,19 +650,44 @@ def train_loop(cfg: dict | None = None) -> Path:
         }
         log["updates"].append(entry)
         log_path.write_text(json.dumps(log, indent=2))
-        print(f"[upd {upd:03d}] reward={entry['mean_reward']:+.3f} "
-              f"V̄={entry['mean_value']:+.3f} loss={stats['loss']:+.3f} "
-              f"kl={stats['kl']:+.4f}  ({elapsed}s)")
 
-        if upd % cfg["checkpoint_every"] == 0:
-            ckpt = LOGS_DIR / f"adapter_{run_name}_u{upd:04d}"
+        bar.set_postfix_str(
+            f"r={entry['mean_reward']:+.3f} "
+            f"V={entry['mean_value']:+.3f} "
+            f"kl={stats['kl']:+.4f} "
+            f"upd={_fmt_dur(elapsed)} "
+            f"ETA={_fmt_dur(eta_s)}")
+        bar.update(1)
+        # Always print a line too so non-tty environments (HF Jobs logs,
+        # nohup) still show progress.
+        print(f"[upd {upd:03d}/{total:03d}] reward={entry['mean_reward']:+.3f} "
+              f"V̄={entry['mean_value']:+.3f} loss={stats['loss']:+.3f} "
+              f"kl={stats['kl']:+.4f}  upd={_fmt_dur(elapsed)} "
+              f"wall={_fmt_dur(wall)}  ETA={_fmt_dur(eta_s)}",
+              flush=True)
+
+        # Stream the JSON log to HF every update so even if the pod dies
+        # mid-step you can recover the metrics.
+        push_ckpt(log_path.parent, "logs")
+
+        if upd % cfg["checkpoint_every"] == 0 or upd == total:
+            ckpt_name = (f"adapter_{run_name}_u{upd:04d}"
+                         if upd != total else f"adapter_{run_name}_final")
+            ckpt = LOGS_DIR / ckpt_name
             actor.model.save_pretrained(str(ckpt))
             actor.tokenizer.save_pretrained(str(ckpt))
-            print(f"[ckpt] saved {ckpt}")
+            print(f"[ckpt] saved {ckpt}", flush=True)
+            push_ckpt(ckpt, ckpt_name)
 
+    bar.close()
     final_ckpt = LOGS_DIR / f"adapter_{run_name}_final"
-    actor.model.save_pretrained(str(final_ckpt))
-    actor.tokenizer.save_pretrained(str(final_ckpt))
+    if not final_ckpt.exists():
+        actor.model.save_pretrained(str(final_ckpt))
+        actor.tokenizer.save_pretrained(str(final_ckpt))
+        push_ckpt(final_ckpt, final_ckpt.name)
+
+    total_wall = time.time() - train_t0
+    print(f"[done] total wall: {_fmt_dur(total_wall)}")
     print(f"[done] final adapter -> {final_ckpt}")
     print(f"[done] JSON log     -> {log_path}")
     return log_path

scripts/HF_JOBS.md

@@ -0,0 +1,73 @@
+# Train on Hugging Face Jobs (real GPU, billed by the second)
+
+Goal: launch a real GPU run from your laptop, watch live progress, and never lose a checkpoint even if the credits run out.
+
+## What you get
+- **Live progress bar + ETA** every PPO update (`[upd 027/120] reward=+0.31 V=+0.18 kl=+0.0042 upd=18s wall=8m12s ETA=27m11s`).
+- **Checkpoints uploaded to HF Hub** every 20 updates (and at the end). If your $30 credit dies mid-run, the latest adapter is already at `https://huggingface.co/<you>/incident-commander-actor`.
+- **Rolling JSON log** also uploaded after every update — you can refresh the HF page and see metrics live.
+
+## One-time setup (on your laptop)
+```powershell
+pip install -U "huggingface_hub[cli]"
+hf auth login                         # paste your hf_… token
+```
+
+## Launch the run
+Pick **one** of these GPU flavors (cheapest → fastest):
+
+| Flavor          | Price  | 120-update run | Notes |
+|-----------------|--------|----------------|-------|
+| `l4x1`          | ~$0.80/hr | ~25–35 min  | **Recommended** — best $/perf. ~$0.50 total. |
+| `a10g-large`    | ~$1.05/hr | ~20–30 min  | Slightly faster, slightly more expensive. |
+| `a100-large`    | ~$3.40/hr | ~8–12 min   | Fastest, biggest dent in credits. |
+
+```powershell
+$env:HF_TOKEN     = "<paste-your-hf_-token-here>"
+$env:IC_PUSH_USER = "sagnik-mukherjee"     # checkpoints land here
+$env:IC_REPO_URL  = "https://github.com/r1cksync/meta-rl-hack.git"
+
+hf jobs run `
+    --flavor l4x1 `
+    --secret HF_TOKEN=$env:HF_TOKEN `
+    --env IC_PUSH_USER=$env:IC_PUSH_USER `
+    --env IC_REPO_URL=$env:IC_REPO_URL `
+    --env IC_TOTAL_UPDATES=120 `
+    --env IC_ROLLOUTS=6 `
+    --env IC_RUN_NAME=hfjob01 `
+    --env HF_HUB_ENABLE_HF_TRANSFER=1 `
+    --image "huggingface/transformers-pytorch-gpu:latest" `
+    -- bash -c "git clone --depth 1 `$IC_REPO_URL /workspace/ic && cd /workspace/ic && bash scripts/hf_job_entrypoint.sh"
+```
+
+> The `--secret` flag injects HF_TOKEN at runtime so it never lands in HF Hub history. The `--env` flags are visible in the job UI but contain no secrets.
+
+## Watch progress
+The `hf jobs run` command streams logs to your terminal. To detach + reattach:
+```powershell
+hf jobs run --detach ... # prints a job ID
+hf jobs logs <job-id> --follow
+hf jobs ps                # list running jobs + spend so far
+```
+
+You'll see a `tqdm` bar plus a per-update line so even non-TTY logs are readable. Look for `ETA=` to know how much wall-time is left.
+
+## Recover if credits die mid-run
+Every 20 updates the adapter is uploaded to:
+```
+https://huggingface.co/sagnik-mukherjee/incident-commander-actor
+```
+under paths like `adapter_hfjob01_u0040/`, `adapter_hfjob01_u0060/`, etc. The training log streams to `logs/training_hfjob01.json` every update. So even if the job is killed at update 67 you still have:
+- the u0060 adapter (LoRA + tokenizer)
+- the partial log up through update 67
+
+To resume locally:
+```powershell
+hf download sagnik-mukherjee/incident-commander-actor adapter_hfjob01_u0060 --local-dir .\resume
+```
+Then point the trainer at the `--resume-from` adapter (planned, not wired yet — for now you'd start fresh from the saved adapter as the actor).
+
+## Cost guardrails
+- The `Qwen2.5-72B-Instruct` critic runs on **HF Inference Providers**, billed against the same $30. Calls are cached, so a 120-update run is typically ~$1–2 in critic spend.
+- Training compute on `l4x1` for ~30 min ≈ **$0.40**.
+- **Total budget**: ~$2–3 for the full real run, leaving ~$27 for re-runs or longer training.

scripts/download_latest_ckpt.py

@@ -0,0 +1,70 @@
+"""Download the latest checkpoint from the HF Hub model repo.
+
+Use this if your HF Jobs run died mid-way and you want the most recent
+adapter on your laptop.
+
+    python scripts/download_latest_ckpt.py --user sagnik-mukherjee
+                                            --run  hfjob01
+                                            --out  ./resume
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--user", required=True,
+                    help="HF username (e.g. sagnik-mukherjee)")
+    ap.add_argument("--run",  default=None,
+                    help="Run name (e.g. hfjob01). If omitted, picks latest.")
+    ap.add_argument("--out",  default="./resume",
+                    help="Local directory to write the adapter into.")
+    ap.add_argument("--prefer-final", action="store_true",
+                    help="Pick *_final if present, else newest u#### dir.")
+    args = ap.parse_args()
+
+    if not os.environ.get("HF_TOKEN"):
+        sys.exit("HF_TOKEN env var required.")
+
+    from huggingface_hub import HfApi, snapshot_download
+
+    repo = f"{args.user}/incident-commander-actor"
+    api  = HfApi(token=os.environ["HF_TOKEN"])
+    print(f"Listing files in {repo} …")
+    files = api.list_repo_files(repo, repo_type="model")
+
+    # Top-level checkpoint dirs look like  adapter_<run>_u0040/...  or
+    #                                       adapter_<run>_final/...
+    prefix = "adapter_"
+    if args.run:
+        prefix = f"adapter_{args.run}_"
+    candidates = sorted({f.split("/")[0] for f in files
+                         if f.startswith(prefix)})
+    if not candidates:
+        sys.exit(f"No checkpoint dirs found under prefix '{prefix}*'.")
+
+    if args.prefer_final and any(c.endswith("_final") for c in candidates):
+        chosen = [c for c in candidates if c.endswith("_final")][-1]
+    else:
+        # Sort so u0120 > u0100 > _final lexically — pick the lexicographically
+        # largest, which works because u#### are zero-padded.
+        chosen = candidates[-1]
+
+    out = Path(args.out) / chosen
+    out.mkdir(parents=True, exist_ok=True)
+    print(f"Downloading {chosen} → {out}")
+    snapshot_download(
+        repo_id=repo, repo_type="model",
+        allow_patterns=[f"{chosen}/*"],
+        local_dir=str(Path(args.out)),
+        token=os.environ["HF_TOKEN"],
+    )
+    print(f"Done. Adapter at: {out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/hf_job_entrypoint.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# HF Jobs entrypoint. Clones the repo, installs deps, runs the real training.
+# Invoked by `hf jobs run` — see scripts/HF_JOBS.md for the exact command.
+set -euo pipefail
+
+REPO_URL="${IC_REPO_URL:-https://github.com/r1cksync/meta-rl-hack.git}"
+WORK="/workspace/incident-commander"
+
+echo "[hfjob] === stage 1: clone ==="
+git clone --depth 1 "$REPO_URL" "$WORK"
+cd "$WORK"
+
+echo "[hfjob] === stage 2: install ==="
+python -m pip install -q --upgrade pip
+python -m pip install -q --no-deps \
+    "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
+python -m pip install -q --no-deps \
+    "xformers<0.0.27" trl peft accelerate bitsandbytes
+python -m pip install -q "huggingface_hub>=0.25" "pydantic>=2,<3" httpx
+
+echo "[hfjob] === stage 3: train ==="
+python scripts/run_training.py
+
+echo "[hfjob] === stage 4: artifacts ==="
+ls -la colab/logs/ || true
+ls -la rl-agent/replays/ || true

scripts/launch_hf_job.ps1

@@ -0,0 +1,47 @@
+# One-line launcher for the HF Jobs training run.
+#
+# Usage:
+#   $env:HF_TOKEN     = "hf_..."           # required
+#   $env:IC_PUSH_USER = "sagnik-mukherjee" # required if you want checkpoints pushed
+#   ./scripts/launch_hf_job.ps1            # uses defaults (l4x1, 120 updates, 6 rollouts)
+#
+# Optional overrides:
+#   $env:IC_GPU_FLAVOR    = "a10g-large"   # default l4x1
+#   $env:IC_TOTAL_UPDATES = 200            # default 120
+#   $env:IC_REPO_URL      = "https://github.com/<you>/<repo>.git"
+
+$ErrorActionPreference = "Stop"
+
+if (-not $env:HF_TOKEN)     { throw "HF_TOKEN env var is required." }
+if (-not $env:IC_PUSH_USER) { Write-Warning "IC_PUSH_USER not set — checkpoints WILL NOT be pushed." }
+
+$flavor   = if ($env:IC_GPU_FLAVOR)    { $env:IC_GPU_FLAVOR }    else { "l4x1" }
+$updates  = if ($env:IC_TOTAL_UPDATES) { $env:IC_TOTAL_UPDATES } else { "120" }
+$rollouts = if ($env:IC_ROLLOUTS)      { $env:IC_ROLLOUTS }      else { "6" }
+$repo     = if ($env:IC_REPO_URL)      { $env:IC_REPO_URL }      else { "https://github.com/r1cksync/meta-rl-hack.git" }
+$run      = if ($env:IC_RUN_NAME)      { $env:IC_RUN_NAME }      else { "hfjob_$(Get-Date -Format yyyyMMdd_HHmm)" }
+
+Write-Host "Launching HF Job:"
+Write-Host "  GPU flavor : $flavor"
+Write-Host "  Run name   : $run"
+Write-Host "  Updates    : $updates × $rollouts rollouts"
+Write-Host "  Repo       : $repo"
+Write-Host "  Push user  : $($env:IC_PUSH_USER)"
+
+$cmd = @(
+    "git clone --depth 1 `$IC_REPO_URL /workspace/ic",
+    "cd /workspace/ic",
+    "bash scripts/hf_job_entrypoint.sh"
+) -join " && "
+
+hf jobs run `
+    --flavor $flavor `
+    --secret "HF_TOKEN=$env:HF_TOKEN" `
+    --env   "IC_PUSH_USER=$env:IC_PUSH_USER" `
+    --env   "IC_REPO_URL=$repo" `
+    --env   "IC_TOTAL_UPDATES=$updates" `
+    --env   "IC_ROLLOUTS=$rollouts" `
+    --env   "IC_RUN_NAME=$run" `
+    --env   "HF_HUB_ENABLE_HF_TRANSFER=1" `
+    --image "huggingface/transformers-pytorch-gpu:latest" `
+    -- bash -c "$cmd"

scripts/run_training.py

@@ -0,0 +1,114 @@
+"""Non-notebook entrypoint for the IncidentCommander RL training run.
+
+Designed to be invoked by `hf jobs run` (or any plain `python` shell). Reads
+configuration from env vars so the same script works in Colab, HF Jobs, and
+local docker.
+
+Required env vars:
+    HF_TOKEN              - HF token with Read scope (Write if pushing adapters)
+
+Optional env vars:
+    IC_TOTAL_UPDATES      - default 120
+    IC_ROLLOUTS           - default 6
+    IC_MAX_STEPS          - default 16
+    IC_RUN_NAME           - default "hfjob01"
+    IC_PUSH_USER          - if set, pushes adapter+logs to <user>/incident-commander-actor
+    IC_CRITIC_MODEL       - default "Qwen/Qwen2.5-72B-Instruct"
+"""
+from __future__ import annotations
+
+import logging
+import os
+import sys
+import warnings
+from pathlib import Path
+
+# Silence the noisy Qwen FutureWarning before importing transformers.
+warnings.filterwarnings("ignore", category=FutureWarning, module="transformers")
+warnings.filterwarnings("ignore", message=".*max_new_tokens.*max_length.*")
+warnings.filterwarnings("ignore", message=".*attention mask API.*")
+logging.getLogger("transformers").setLevel(logging.ERROR)
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "rl-agent"))
+
+# Mock AWS so the simulator runs without boto3 creds.
+os.environ.setdefault("INCIDENT_COMMANDER_MOCK", "true")
+
+if not os.environ.get("HF_TOKEN"):
+    sys.exit("HF_TOKEN env var is required.")
+os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", os.environ["HF_TOKEN"])
+
+from huggingface_hub import login, whoami                       # noqa: E402
+login(os.environ["HF_TOKEN"], add_to_git_credential=False)
+print(f"[hfjob] logged in as {whoami(token=os.environ['HF_TOKEN']).get('name')}")
+
+from colab.train_lib import CFG, train_loop                     # noqa: E402
+
+CFG.update({
+    "total_updates":       int(os.environ.get("IC_TOTAL_UPDATES", 120)),
+    "rollouts_per_update": int(os.environ.get("IC_ROLLOUTS", 6)),
+    "max_steps_per_ep":    int(os.environ.get("IC_MAX_STEPS", 16)),
+    "checkpoint_every":    int(os.environ.get("IC_CKPT_EVERY", 20)),
+    "critic_provider":     "hf",
+    "critic_model":        os.environ.get("IC_CRITIC_MODEL",
+                                          "Qwen/Qwen2.5-72B-Instruct"),
+    "lr":                  1e-5,
+    "kl_coef":             0.02,
+    "clip_eps":            0.20,
+    "gae_lambda":          0.92,
+    "run_name":            os.environ.get("IC_RUN_NAME", "hfjob01"),
+    "tasks": [
+        "sim_easy_lambda_throttle_001", "sim_easy_lambda_throttle_010",
+        "sim_med_eb_lambda_016",       "sim_med_eb_lambda_021",
+        "sim_hard_apigw_chain_001",    "sim_hard_ddb_chain_021",
+        "sim_hard_iam_chain_011",
+        "sim_advanced_cascade_users_db_001",
+        "sim_advanced_runbook_trap_postgres_001",
+        "sim_advanced_trolley_orders_db_001",
+        "sim_advanced_saboteur_duel_001",
+        "sim_advanced_slack_redherring_001",
+        "sim_gen_app_leak_checkout_007",   "sim_gen_app_leak_payments_019",
+        "sim_gen_db_duel_users_db_003",    "sim_gen_db_duel_orders_db_015",
+        "sim_gen_redherring_payments_013", "sim_gen_redherring_auth_001",
+        "sim_gen_cascade_payments_db_004", "sim_gen_cascade_users_db_023",
+        "sim_gen_cache_warm_session_cache_004",
+        "sim_gen_peak_frontend_001",
+        "sim_gen_restore_payments_db_001",
+    ],
+})
+
+print(f"[hfjob] starting run '{CFG['run_name']}': "
+      f"{CFG['total_updates']}×{CFG['rollouts_per_update']}×{CFG['max_steps_per_ep']} "
+      f"(~{CFG['total_updates'] * CFG['rollouts_per_update'] * CFG['max_steps_per_ep']:,} transitions)")
+
+log_path = train_loop()
+print(f"[hfjob] training log → {log_path}")
+
+# ── Optional: push artifacts to a HF model repo ─────────────────────────
+push_user = os.environ.get("IC_PUSH_USER", "").strip()
+if push_user:
+    import glob
+    from huggingface_hub import HfApi, create_repo
+    api  = HfApi(token=os.environ["HF_TOKEN"])
+    repo = f"{push_user}/incident-commander-actor"
+    create_repo(repo, exist_ok=True, repo_type="model",
+                token=os.environ["HF_TOKEN"])
+    finals = sorted(glob.glob(str(ROOT / "colab" / "logs" / "adapter_*_final")))
+    if finals:
+        api.upload_folder(folder_path=finals[-1], repo_id=repo,
+                          repo_type="model", path_in_repo="adapter")
+    api.upload_folder(folder_path=str(ROOT / "colab" / "logs"),
+                      repo_id=repo, repo_type="model", path_in_repo="logs",
+                      allow_patterns=["*.json"])
+    replay_dir = ROOT / "rl-agent" / "replays"
+    if replay_dir.exists():
+        api.upload_folder(folder_path=str(replay_dir), repo_id=repo,
+                          repo_type="model", path_in_repo="replays",
+                          allow_patterns=["*.html"])
+    print(f"[hfjob] pushed → https://huggingface.co/{repo}")
+else:
+    print("[hfjob] IC_PUSH_USER not set — skipping HF push.")
+
+print("[hfjob] done.")