Upload folder using huggingface_hub

.claude/settings.local.json

@@ -0,0 +1,21 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(env)",
+      "Bash(curl -s -o /dev/null -w \"HTTP %{http_code}  time=%{time_total}s\\\\n\" --max-time 5 http://nexus.sii.shaipower.online/repository/pypi/simple/)",
+      "Bash(curl -s -o /dev/null -w \"HTTPS %{http_code}  time=%{time_total}s\\\\n\" --max-time 5 https://nexus.sii.shaipower.online/repository/pypi/simple/)",
+      "Bash(curl -s -o /dev/null -w \"HTTP %{http_code}  time=%{time_total}s\\\\n\" --max-time 15 http://nexus.sii.shaipower.online/repository/pypi/simple/)",
+      "Bash(python3 -c \"import sys, json; d=json.loads\\(sys.stdin.read\\(\\)\\); print\\(json.dumps\\(d, indent=2, ensure_ascii=False\\)[:3000]\\)\")",
+      "Bash(python3 -c ' *)",
+      "Bash(nvidia-smi)",
+      "Bash(python3 *)",
+      "Read(//root/miniconda3/envs/**)",
+      "Bash(conda env *)",
+      "Bash(/root/miniconda3/envs/verl/bin/python *)",
+      "Bash(/root/miniconda3/envs/verl/bin/pip list *)",
+      "Bash(chmod +x *)",
+      "Bash(/root/miniconda3/envs/verl/bin/pip show *)",
+      "Bash(grep -E \"\\\\.\\(py|sh\\)$\")"
+    ]
+  }
+}

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+verl/core.778099 filter=lfs diff=lfs merge=lfs -text
+verl/verl/trainer/ppo/__pycache__/core_algos.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text

.sii/ide-port

@@ -0,0 +1 @@
+30003

0519_sft_test/data/convert_to_parquet.py

@@ -0,0 +1,154 @@
+# Convert ind_v4 SFT JSONL into a verl-compatible parquet for MultiTurnSFTDataset.
+#
+# Source format (per line):
+#   {
+#     "messages": [
+#       {"role": "user",      "content": "<image>\n..."},
+#       {"role": "assistant", "content": "", "tool_calls": [{"id": "...", "type": "function",
+#                                                            "function": {"name": "...",
+#                                                                         "arguments": "{...}"}}]}
+#     ],
+#     "images": ["/abs/path/to/img.png"],
+#     "tools":  [{"type": "function", "function": {...}}],
+#     "metadata": {...}
+#   }
+#
+# Target parquet columns (verl MultiTurnSFTDataset):
+#   - messages: list[dict]
+#   - images:   list[{"bytes": <png bytes>}]   (loadable by qwen_vl_utils.fetch_image)
+#   - tools:    list[dict]
+
+import argparse
+import json
+import os
+import random
+from pathlib import Path
+
+import pandas as pd
+
+
+def load_jsonl(path):
+    rows = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def read_image_bytes(path: str) -> bytes:
+    with open(path, "rb") as f:
+        return f.read()
+
+
+def normalize_messages(messages):
+    out = []
+    for m in messages:
+        nm = {"role": m["role"], "content": m.get("content", "") or ""}
+        if "tool_calls" in m and m["tool_calls"]:
+            tcs = []
+            for tc in m["tool_calls"]:
+                fn = tc.get("function", {})
+                args = fn.get("arguments", "")
+                if not isinstance(args, str):
+                    args = json.dumps(args, ensure_ascii=False)
+                tcs.append(
+                    {
+                        "id": tc.get("id", ""),
+                        "type": tc.get("type", "function"),
+                        "function": {
+                            "name": fn.get("name", ""),
+                            "arguments": args,
+                        },
+                    }
+                )
+            nm["tool_calls"] = tcs
+        out.append(nm)
+    return out
+
+
+def convert(rows, image_root: str | None, max_samples: int = -1):
+    if max_samples > 0:
+        rows = rows[:max_samples]
+    out = []
+    for row in rows:
+        messages = normalize_messages(row["messages"])
+
+        images = []
+        for img_path in row.get("images", []) or []:
+            p = img_path
+            if image_root and not os.path.isabs(p):
+                p = os.path.join(image_root, p)
+            images.append({"bytes": read_image_bytes(p)})
+
+        tools = row.get("tools", []) or []
+
+        out.append(
+            {
+                "messages": messages,
+                "images": images,
+                "tools": tools,
+            }
+        )
+    return out
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--src",
+        default="/inspire/qb-ilm2/project/26summer-camp-21/26210880/0518_dataBuilder_liubw"
+        "/in_domain/output/ind_v4/sft_rl/sft_train.jsonl",
+        help="Path to source SFT JSONL.",
+    )
+    parser.add_argument(
+        "--image_root",
+        default=None,
+        help="Root for relative image paths. JSONL already has absolute paths, leave None.",
+    )
+    parser.add_argument(
+        "--out_dir",
+        default=str(Path(__file__).resolve().parent),
+        help="Directory to write {train,val}.parquet.",
+    )
+    parser.add_argument("--val_ratio", type=float, default=0.02)
+    parser.add_argument("--max_samples", type=int, default=-1)
+    parser.add_argument("--seed", type=int, default=42)
+    args = parser.parse_args()
+
+    out_dir = Path(args.out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    rows = load_jsonl(args.src)
+    print(f"loaded {len(rows)} rows from {args.src}")
+
+    rng = random.Random(args.seed)
+    rng.shuffle(rows)
+
+    if args.max_samples > 0:
+        rows = rows[: args.max_samples]
+        print(f"truncated to {len(rows)} samples")
+
+    n_val = max(1, int(len(rows) * args.val_ratio)) if args.val_ratio > 0 else 0
+    val_rows = rows[:n_val]
+    train_rows = rows[n_val:]
+    print(f"split: train={len(train_rows)}, val={len(val_rows)}")
+
+    train_records = convert(train_rows, args.image_root)
+    val_records = convert(val_rows, args.image_root) if val_rows else []
+
+    train_df = pd.DataFrame(train_records)
+    train_path = out_dir / "train.parquet"
+    train_df.to_parquet(train_path, index=False)
+    print(f"wrote {train_path} ({len(train_df)} rows)")
+
+    if val_records:
+        val_df = pd.DataFrame(val_records)
+        val_path = out_dir / "val.parquet"
+        val_df.to_parquet(val_path, index=False)
+        print(f"wrote {val_path} ({len(val_df)} rows)")
+
+
+if __name__ == "__main__":
+    main()

0519_sft_test/data/train.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e61cc170fa7c579bb897cfca7ec2058ead67bfbab7e3c7830b80211958737d4c
+size 131141435

0519_sft_test/data/val.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7741860c3e5aadef4589d9306a95495c232b91798e4f1cc3ddcde799b03b48e7
+size 2522091

0519_sft_test/run_sft_qwen3vl_8b.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# SFT for Qwen3-VL-8B-Thinking on the ind_v4 dataset (verl FSDP engine).
+#
+# Usage:
+#   bash run_sft_qwen3vl_8b.sh                     # default 8 GPUs
+#   NUM_TRAINERS=4 bash run_sft_qwen3vl_8b.sh      # override GPU count
+#   SP_SIZE=2 NUM_TRAINERS=8 bash run_sft_qwen3vl_8b.sh
+set -xeuo pipefail
+
+WORKDIR=$(cd "$(dirname "$0")" && pwd)
+
+MODEL_PATH=${MODEL_PATH:-/inspire/qb-ilm2/project/26summer-camp-21/26210880/0518_test_liubw/models/Qwen3-VL-8B-Thinking}
+TRAIN_FILES=${TRAIN_FILES:-${WORKDIR}/data/train.parquet}
+VAL_FILES=${VAL_FILES:-${WORKDIR}/data/val.parquet}
+
+NUM_TRAINERS=${NUM_TRAINERS:-8}
+SP_SIZE=${SP_SIZE:-2}                # ulysses sequence parallel
+FSDP_SIZE=${FSDP_SIZE:--1}
+FSDP_STRATEGY=${FSDP_STRATEGY:-fsdp2}
+
+TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-32}
+MICRO_BATCH_SIZE_PER_GPU=${MICRO_BATCH_SIZE_PER_GPU:-1}
+MAX_LENGTH=${MAX_LENGTH:-8192}
+MAX_TOKEN_LEN_PER_GPU=${MAX_TOKEN_LEN_PER_GPU:-16384}
+TOTAL_EPOCHS=${TOTAL_EPOCHS:-3}
+LR=${LR:-1e-5}
+
+PROJECT_NAME=${PROJECT_NAME:-ind_v4_sft}
+EXPERIMENT_NAME=${EXPERIMENT_NAME:-qwen3vl-8b-thinking-fsdp${FSDP_STRATEGY}-sp${SP_SIZE}}
+SAVE_DIR=${SAVE_DIR:-${WORKDIR}/checkpoints/${PROJECT_NAME}/${EXPERIMENT_NAME}}
+LOGGER=${LOGGER:-"['console']"}      # set to "['console','wandb']" once wandb is logged in
+RESUME_MODE=${RESUME_MODE:-auto}
+
+mkdir -p "${SAVE_DIR}"
+
+cd /inspire/qb-ilm2/project/26summer-camp-21/26210880/0519_verl/verl
+
+torchrun --standalone --nnodes=1 --nproc-per-node="${NUM_TRAINERS}" \
+    -m verl.trainer.sft_trainer \
+    data.train_files="${TRAIN_FILES}" \
+    data.val_files="${VAL_FILES}" \
+    data.train_batch_size="${TRAIN_BATCH_SIZE}" \
+    data.micro_batch_size_per_gpu="${MICRO_BATCH_SIZE_PER_GPU}" \
+    data.max_length="${MAX_LENGTH}" \
+    data.max_token_len_per_gpu="${MAX_TOKEN_LEN_PER_GPU}" \
+    data.use_dynamic_bsz=True \
+    data.pad_mode=no_padding \
+    data.truncation=right \
+    data.messages_key=messages \
+    data.tools_key=tools \
+    data.ignore_input_ids_mismatch=True \
+    model.path="${MODEL_PATH}" \
+    model.use_remove_padding=True \
+    engine=fsdp \
+    optim=fsdp \
+    optim.lr="${LR}" \
+    optim.lr_warmup_steps_ratio=0.03 \
+    optim.weight_decay=0.1 \
+    optim.betas="[0.9,0.95]" \
+    optim.clip_grad=1.0 \
+    optim.min_lr_ratio=0.1 \
+    optim.warmup_style=cosine \
+    engine.ulysses_sequence_parallel_size="${SP_SIZE}" \
+    engine.strategy="${FSDP_STRATEGY}" \
+    engine.fsdp_size="${FSDP_SIZE}" \
+    trainer.default_local_dir="${SAVE_DIR}" \
+    trainer.project_name="${PROJECT_NAME}" \
+    trainer.experiment_name="${EXPERIMENT_NAME}" \
+    trainer.total_epochs="${TOTAL_EPOCHS}" \
+    trainer.logger="${LOGGER}" \
+    trainer.save_freq=500 \
+    trainer.test_freq=200 \
+    trainer.max_ckpt_to_keep=3 \
+    trainer.resume_mode="${RESUME_MODE}" \
+    checkpoint.save_contents=[model,optimizer,extra]

0520_rej_test/README.md

@@ -0,0 +1,69 @@
+# 0520_rej_test — Offline RAFT/RFT framework
+
+verl 主仓库**没有现成的 reward-based 拒绝采样训练循环**：
+- `algorithm.filter_groups` 只有 dataclass，没有 trainer 消费
+- `algorithm.rollout_correction` 是 IS-based 拒绝，依据是重要性比值，不是 reward
+
+这里搭的是经典 **RAFT (Reward-rAnked Fine-Tuning)** —— 完全离线、字面意义上的拒绝采样：
+对当前模型采样 N 条 → reward 打分 → 取 top-K → 喂给 SFT。
+
+```
+rl_train.jsonl
+   ↓ build_prompts.py
+prompts.parquet
+   ↓ Stage A: verl.trainer.main_generation_server (vLLM, n=N)
+gen.parquet
+   ↓ Stage B: reward/score.py + reward/reward_fn.py (STUB)
+scored.parquet
+   ↓ Stage C: select/topk_to_sft.py
+raft_train.parquet
+   ↓ Stage D: scripts/run_sft.sh -> 0519_sft_test/run_sft_qwen3vl_8b.sh
+checkpoints/
+```
+
+## 现状
+- 框架完整可执行；
+- **`reward/reward_fn.py` 是 stub**，固定 `compute_reward(prompt, response, metadata, tools, ...)` 签名，永远返回 `0.0`，等后续实现；
+- 不动 `0519_sft_test/` 任何文件，Stage D 通过环境变量复用其 SFT 脚本。
+
+## 使用
+
+整体跑：
+```bash
+bash run_pipeline.sh
+```
+
+只跑某一段：
+```bash
+STAGE=A_pre bash run_pipeline.sh   # 只生成 prompts.parquet
+STAGE=A     bash run_pipeline.sh   # 跑 vLLM 采样
+STAGE=B     bash run_pipeline.sh   # 打分
+STAGE=C     bash run_pipeline.sh   # 选 top-K + 转 SFT 格式
+STAGE=D     bash run_pipeline.sh   # SFT
+```
+
+Smoke 试跑（4 条 prompt × 2 个 sample）：
+```bash
+MAX_SAMPLES=4 N_SAMPLES=2 RESPONSE_LENGTH=512 STAGE=A_pre bash run_pipeline.sh
+MAX_SAMPLES=4 N_SAMPLES=2 RESPONSE_LENGTH=512 STAGE=A     bash run_pipeline.sh
+STAGE=B bash run_pipeline.sh
+TOP_K=1 STAGE=C bash run_pipeline.sh
+```
+
+## 关键依赖
+
+| 用途 | 路径 |
+|---|---|
+| Stage A 入口 | `verl/trainer/main_generation_server.py` |
+| 多模态 chat template | `verl/workers/rollout/schemas.py` |
+| SFT 目标 schema | `verl/utils/dataset/multiturn_sft_dataset.py` |
+| SFT 启动脚本（被复用） | `0519_sft_test/run_sft_qwen3vl_8b.sh` |
+| 模型 | `0518_test_liubw/models/Qwen3-VL-8B-Thinking` |
+| RL 原始数据 | `0518_dataBuilder_liubw/.../sft_rl/rl_train.jsonl` |
+
+## 设计要点
+
+- **图像如何进 vLLM？** verl 的 `main_generation_server` 直接把 `prompt` 列里的 OpenAI 格式 messages 转给 vLLM 的 chat completions 接口 —— 我们把每张图编成 `data:image/png;base64,...` URL 塞进 multimodal content block。Stage C 再从 `image_paths` 重新读字节进 SFT parquet。
+- **tools 怎么走？** `main_generation_server` 不读 `tools` 列；`tools` 的 JSON schema 当作文本前缀拼进 user turn。模型是 Qwen3-VL-Thinking，训练时见过 `<tool_call>{...}</tool_call>` 输出格式，会自然产出。
+- **怎么把模型输出还原成结构化 tool_calls？** Stage C 用正则提 `<tool_call>` 块 → `json.loads` → 构造 OpenAI tool_call dict（`arguments` 强制成字符串，与 `0519_sft_test/data/convert_to_parquet.py` 一致）。解析失败的样本直接丢。
+- **SFT schema 完全和 0519_sft_test 一致**，所以可以直接复用其 dataset 加载 + chat template 验证路径。

0520_rej_test/data/build_prompts.py

@@ -0,0 +1,135 @@
+"""Build prompts.parquet for Stage A (verl.trainer.main_generation_server).
+
+Source: ind_v4 rl_train.jsonl with shape
+  {messages, images, tools, metadata}
+
+main_generation_server expects a single `prompt` column whose value is an
+OpenAI-style chat-completions `messages` list. It does NOT read `tools` or
+`images` columns directly — both must be baked into `prompt`.
+
+Encoding:
+  - `images[i]` is read from disk and embedded as a base64 data-URL inside
+    a multimodal content block on the user turn.
+  - `tools` (the JSON spec) is rendered as a textual preamble prepended to
+    the user turn so that even without server-side tool injection the model
+    sees the tool name + schema. The model is Qwen3-VL-Thinking, which is
+    trained to emit `<tool_call>{...}</tool_call>` blocks — Stage C will
+    parse them back out.
+
+Extra columns kept on the parquet so downstream stages can use them:
+  - `metadata`: original metadata dict (test_points, expression_numpy, ...)
+  - `image_paths`: original image paths (so Stage C can re-read bytes
+                   without going through the data-URL).
+  - `tools_orig`: original tools spec (for Stage C's SFT reconstruction).
+"""
+
+import argparse
+import base64
+import json
+import mimetypes
+import os
+import random
+from pathlib import Path
+
+import pandas as pd
+
+
+def load_jsonl(path):
+    rows = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def img_to_data_url(path: str) -> str:
+    mime, _ = mimetypes.guess_type(path)
+    if mime is None:
+        mime = "image/png"
+    with open(path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode("ascii")
+    return f"data:{mime};base64,{b64}"
+
+
+def render_tools_preamble(tools: list[dict]) -> str:
+    if not tools:
+        return ""
+    lines = ["You have access to the following tools. Reply via a single "
+             "<tool_call>{...}</tool_call> block whose JSON contains "
+             "`name` and `arguments`."]
+    for t in tools:
+        lines.append(json.dumps(t, ensure_ascii=False))
+    return "\n".join(lines) + "\n\n"
+
+
+def build_prompt(row: dict) -> list[dict]:
+    """Construct OpenAI multimodal `messages` from one raw RL sample."""
+    user_text = row["messages"][0]["content"]
+    if user_text.startswith("<image>\n"):
+        user_text = user_text[len("<image>\n") :]
+    elif user_text.startswith("<image>"):
+        user_text = user_text[len("<image>") :]
+
+    preamble = render_tools_preamble(row.get("tools") or [])
+    text_block = preamble + user_text
+
+    content = []
+    for img_path in row.get("images") or []:
+        content.append(
+            {"type": "image_url",
+             "image_url": {"url": img_to_data_url(img_path)}}
+        )
+    content.append({"type": "text", "text": text_block})
+
+    return [{"role": "user", "content": content}]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--src",
+        default="/inspire/qb-ilm2/project/26summer-camp-21/26210880"
+        "/0518_dataBuilder_liubw/in_domain/output/ind_v4/sft_rl/rl_train.jsonl",
+    )
+    parser.add_argument(
+        "--out",
+        default=str(Path(__file__).resolve().parent / "prompts.parquet"),
+    )
+    parser.add_argument("--max_samples", type=int, default=-1)
+    parser.add_argument("--shuffle", action="store_true")
+    parser.add_argument("--seed", type=int, default=42)
+    args = parser.parse_args()
+
+    rows = load_jsonl(args.src)
+    print(f"loaded {len(rows)} rows from {args.src}")
+
+    if args.shuffle:
+        rng = random.Random(args.seed)
+        rng.shuffle(rows)
+
+    if args.max_samples > 0:
+        rows = rows[: args.max_samples]
+        print(f"truncated to {len(rows)}")
+
+    records = []
+    for row in rows:
+        records.append(
+            {
+                "prompt": build_prompt(row),
+                "metadata": row.get("metadata", {}),
+                "image_paths": row.get("images") or [],
+                "tools_orig": row.get("tools") or [],
+            }
+        )
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    df = pd.DataFrame(records)
+    df.to_parquet(out_path, index=False)
+    print(f"wrote {out_path} ({len(df)} rows)")
+
+
+if __name__ == "__main__":
+    main()

0520_rej_test/reward/reward_fn.py

@@ -0,0 +1,34 @@
+"""Stage B reward function.
+
+This is a STUB. The signature is fixed; downstream `score.py` calls
+`compute_reward(...)` row-wise by keyword. Implement the body later.
+
+Inputs (kwargs):
+  prompt:    list[dict]                   # OpenAI chat-completions messages used at generation
+  response:  str                          # one model completion (raw text)
+  metadata:  dict                         # original sample metadata (test_points, expression_numpy, ...)
+  tools:     list[dict]                   # original `tools` JSON spec
+  index:     int                          # sample index (for logging)
+  sample:    int                          # which of the N samples (0..N-1)
+
+Returns:
+  float in [0, 1]   -- higher is better
+"""
+
+from __future__ import annotations
+
+
+def compute_reward(
+    *,
+    prompt: list[dict],
+    response: str,
+    metadata: dict,
+    tools: list[dict],
+    index: int = 0,
+    sample: int = 0,
+) -> float:
+    # TODO: parse <tool_call>{"name": "submit_expression", "arguments": "..."}</tool_call>
+    #       eval expression at metadata["test_points"], compute MSE-based reward.
+    #       Return 0.0 if parse fails or eval errors out.
+    del prompt, response, metadata, tools, index, sample
+    return 0.0

0520_rej_test/reward/score.py

@@ -0,0 +1,80 @@
+"""Stage B driver: read gen.parquet, score each (prompt, response) pair via
+reward_fn.compute_reward, write scored.parquet with a new `scores` column
+(parallel to `responses`).
+
+Reward fn lives in reward/reward_fn.py and can be swapped via --reward_fn.
+"""
+
+import argparse
+import importlib.util
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+
+def load_reward_fn(path: str, name: str = "compute_reward"):
+    spec = importlib.util.spec_from_file_location("reward_fn_module", path)
+    mod = importlib.util.module_from_spec(spec)
+    sys.modules["reward_fn_module"] = mod
+    assert spec.loader is not None
+    spec.loader.exec_module(mod)
+    fn = getattr(mod, name)
+    return fn
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gen",
+        default=str(Path(__file__).resolve().parent.parent / "data" / "gen.parquet"),
+    )
+    parser.add_argument(
+        "--out",
+        default=str(Path(__file__).resolve().parent.parent / "data" / "scored.parquet"),
+    )
+    parser.add_argument(
+        "--reward_fn",
+        default=str(Path(__file__).resolve().parent / "reward_fn.py"),
+    )
+    parser.add_argument("--reward_name", default="compute_reward")
+    args = parser.parse_args()
+
+    df = pd.read_parquet(args.gen)
+    print(f"loaded {len(df)} rows from {args.gen}; columns: {list(df.columns)}")
+
+    if "responses" not in df.columns:
+        raise SystemExit("expected `responses` column from Stage A; not found.")
+
+    compute_reward = load_reward_fn(args.reward_fn, args.reward_name)
+
+    all_scores = []
+    for idx, row in df.iterrows():
+        prompt = row["prompt"]
+        responses = list(row["responses"])
+        metadata = row.get("metadata", {}) or {}
+        tools = list(row.get("tools_orig", []) or [])
+        scores = []
+        for s_idx, resp in enumerate(responses):
+            score = compute_reward(
+                prompt=prompt,
+                response=resp,
+                metadata=metadata,
+                tools=tools,
+                index=int(idx),
+                sample=s_idx,
+            )
+            scores.append(float(score))
+        all_scores.append(scores)
+
+    df = df.copy()
+    df["scores"] = all_scores
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_parquet(out_path, index=False)
+    print(f"wrote {out_path} ({len(df)} rows)")
+
+
+if __name__ == "__main__":
+    main()

0520_rej_test/run_pipeline.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# End-to-end RAFT pipeline: build prompts -> generate -> score -> top-K -> SFT.
+# Each stage is overrideable via env vars and idempotent.
+set -xeuo pipefail
+
+WORKDIR=$(cd "$(dirname "$0")" && pwd)
+PY=${PY:-python}
+
+MAX_SAMPLES=${MAX_SAMPLES:--1}
+N_SAMPLES=${N_SAMPLES:-8}
+TOP_K=${TOP_K:-1}
+
+STAGE=${STAGE:-all}                    # all | A | B | C | D
+
+if [[ "${STAGE}" == "all" || "${STAGE}" == "A_pre" || "${STAGE}" == "A" ]]; then
+    "${PY}" "${WORKDIR}/data/build_prompts.py" \
+        --out "${WORKDIR}/data/prompts.parquet" \
+        --max_samples "${MAX_SAMPLES}"
+fi
+
+if [[ "${STAGE}" == "all" || "${STAGE}" == "A" ]]; then
+    N_SAMPLES="${N_SAMPLES}" \
+    PROMPTS="${WORKDIR}/data/prompts.parquet" \
+    OUT="${WORKDIR}/data/gen.parquet" \
+    bash "${WORKDIR}/scripts/run_generate.sh"
+fi
+
+if [[ "${STAGE}" == "all" || "${STAGE}" == "B" ]]; then
+    "${PY}" "${WORKDIR}/reward/score.py" \
+        --gen "${WORKDIR}/data/gen.parquet" \
+        --out "${WORKDIR}/data/scored.parquet"
+fi
+
+if [[ "${STAGE}" == "all" || "${STAGE}" == "C" ]]; then
+    "${PY}" "${WORKDIR}/select/topk_to_sft.py" \
+        --scored "${WORKDIR}/data/scored.parquet" \
+        --out "${WORKDIR}/data/raft_train.parquet" \
+        --k "${TOP_K}"
+fi
+
+if [[ "${STAGE}" == "all" || "${STAGE}" == "D" ]]; then
+    bash "${WORKDIR}/scripts/run_sft.sh"
+fi

0520_rej_test/scripts/run_generate.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Stage A: rollout N completions per prompt with vLLM, via verl's
+# main_generation_server.
+#
+# Usage:
+#   bash run_generate.sh                                  # full prompts.parquet
+#   PROMPTS=/path/to/sub.parquet OUT=/tmp/gen.parquet \
+#       N_SAMPLES=2 bash run_generate.sh                  # smoke
+set -xeuo pipefail
+
+WORKDIR=$(cd "$(dirname "$0")/.." && pwd)
+VERL_ROOT=/inspire/qb-ilm2/project/26summer-camp-21/26210880/0519_verl/verl
+
+MODEL_PATH=${MODEL_PATH:-/inspire/qb-ilm2/project/26summer-camp-21/26210880/0518_test_liubw/models/Qwen3-VL-8B-Thinking}
+PROMPTS=${PROMPTS:-${WORKDIR}/data/prompts.parquet}
+OUT=${OUT:-${WORKDIR}/data/gen.parquet}
+
+NNODES=${NNODES:-1}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+ROLLOUT_TP=${ROLLOUT_TP:-2}
+ROLLOUT_GPU_MEM_UTIL=${ROLLOUT_GPU_MEM_UTIL:-0.85}
+
+PROMPT_LENGTH=${PROMPT_LENGTH:-4096}
+RESPONSE_LENGTH=${RESPONSE_LENGTH:-4096}
+N_SAMPLES=${N_SAMPLES:-8}
+TEMPERATURE=${TEMPERATURE:-1.0}
+TOP_P=${TOP_P:-0.95}
+
+mkdir -p "$(dirname "${OUT}")"
+
+cd "${VERL_ROOT}"
+
+python3 -m verl.trainer.main_generation_server \
+    trainer.nnodes="${NNODES}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    data.train_files="${PROMPTS}" \
+    data.prompt_key=prompt \
+    +data.output_path="${OUT}" \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.model.trust_remote_code=True \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.temperature="${TEMPERATURE}" \
+    actor_rollout_ref.rollout.top_p="${TOP_P}" \
+    actor_rollout_ref.rollout.prompt_length="${PROMPT_LENGTH}" \
+    actor_rollout_ref.rollout.response_length="${RESPONSE_LENGTH}" \
+    actor_rollout_ref.rollout.tensor_model_parallel_size="${ROLLOUT_TP}" \
+    actor_rollout_ref.rollout.gpu_memory_utilization="${ROLLOUT_GPU_MEM_UTIL}" \
+    actor_rollout_ref.rollout.n="${N_SAMPLES}" "$@"

0520_rej_test/scripts/run_sft.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Stage D wrapper: re-use 0519_sft_test/run_sft_qwen3vl_8b.sh against the
+# RAFT-mined parquet, without modifying the original script.
+set -xeuo pipefail
+
+WORKDIR=$(cd "$(dirname "$0")/.." && pwd)
+
+TRAIN_FILES=${TRAIN_FILES:-${WORKDIR}/data/raft_train.parquet}
+VAL_FILES=${VAL_FILES:-${WORKDIR}/data/raft_val.parquet}
+
+PROJECT_NAME=${PROJECT_NAME:-ind_v4_raft}
+EXPERIMENT_NAME=${EXPERIMENT_NAME:-qwen3vl-8b-thinking-raft}
+SAVE_DIR=${SAVE_DIR:-${WORKDIR}/checkpoints/${PROJECT_NAME}/${EXPERIMENT_NAME}}
+
+TRAIN_FILES="${TRAIN_FILES}" \
+VAL_FILES="${VAL_FILES}" \
+PROJECT_NAME="${PROJECT_NAME}" \
+EXPERIMENT_NAME="${EXPERIMENT_NAME}" \
+SAVE_DIR="${SAVE_DIR}" \
+bash /inspire/qb-ilm2/project/26summer-camp-21/26210880/0519_verl/0519_sft_test/run_sft_qwen3vl_8b.sh "$@"

0520_rej_test/select/topk_to_sft.py

@@ -0,0 +1,148 @@
+"""Stage C: scored.parquet -> raft_train.parquet (verl SFT schema).
+
+For each row pick the top-K responses by score, parse Qwen3-VL-Thinking
+`<tool_call>{...}</tool_call>` blocks back into structured tool_calls, and
+emit one SFT example per (row, kept response). Output schema matches
+`verl/utils/dataset/multiturn_sft_dataset.py`:
+  - messages: [user_turn, assistant_turn_with_tool_calls]
+  - images:   [{"bytes": <png>}]   (re-read from row["image_paths"])
+  - tools:    original tools spec
+"""
+
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+
+import pandas as pd
+
+TOOL_CALL_RE = re.compile(r"<tool_call>\s*(.*?)\s*</tool_call>", re.DOTALL)
+
+
+def read_image_bytes(path: str) -> bytes:
+    with open(path, "rb") as f:
+        return f.read()
+
+
+def parse_tool_calls(text: str) -> tuple[str, list[dict]]:
+    """Strip <tool_call>...</tool_call> blocks out of `text`.
+
+    Returns:
+      (residual_text_with_blocks_removed, list_of_tool_call_dicts)
+
+    Each tool_call dict is OpenAI-style:
+      {"id": "...", "type": "function",
+       "function": {"name": "...", "arguments": "<json string>"}}
+    Failed JSON parses make the whole sample invalid (caller should drop).
+    """
+    tool_calls = []
+    for i, m in enumerate(TOOL_CALL_RE.finditer(text)):
+        raw = m.group(1).strip()
+        try:
+            obj = json.loads(raw)
+        except json.JSONDecodeError:
+            return text, []
+        name = obj.get("name", "")
+        args = obj.get("arguments", obj.get("parameters", {}))
+        if not isinstance(args, str):
+            args = json.dumps(args, ensure_ascii=False)
+        tool_calls.append(
+            {
+                "id": f"call_{i}",
+                "type": "function",
+                "function": {"name": name, "arguments": args},
+            }
+        )
+    residual = TOOL_CALL_RE.sub("", text).strip()
+    return residual, tool_calls
+
+
+def reconstruct_user_turn(prompt_messages: list[dict]) -> dict:
+    """The prompt for generation was multimodal (image_url + text). For SFT
+    we replace it with the conventional `<image>\\n<text>` Qwen format —
+    this matches the existing 0519_sft_test/data parquet exactly. The image
+    bytes are attached separately via the `images` column.
+    """
+    user_msg = prompt_messages[0]
+    text_parts = []
+    for blk in user_msg["content"]:
+        if isinstance(blk, dict) and blk.get("type") == "text":
+            text_parts.append(blk["text"])
+    text = "\n".join(text_parts)
+    return {"role": "user", "content": "<image>\n" + text}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--scored",
+        default=str(Path(__file__).resolve().parent.parent / "data" / "scored.parquet"),
+    )
+    parser.add_argument(
+        "--out",
+        default=str(Path(__file__).resolve().parent.parent / "data" / "raft_train.parquet"),
+    )
+    parser.add_argument("--k", type=int, default=1, help="Top-K responses per prompt.")
+    parser.add_argument("--min_score", type=float, default=None)
+    parser.add_argument("--keep_above_mean", action="store_true",
+                        help="Keep responses scoring strictly above the per-row mean.")
+    args = parser.parse_args()
+
+    df = pd.read_parquet(args.scored)
+    print(f"loaded {len(df)} rows from {args.scored}")
+
+    out_records = []
+    n_dropped_parse = 0
+    n_dropped_score = 0
+    for _, row in df.iterrows():
+        responses = list(row["responses"])
+        scores = list(row["scores"])
+        assert len(responses) == len(scores)
+
+        ranked = sorted(zip(scores, responses), key=lambda x: x[0], reverse=True)
+        if args.keep_above_mean:
+            mean_s = sum(scores) / max(1, len(scores))
+            ranked = [(s, r) for s, r in ranked if s > mean_s]
+        if args.min_score is not None:
+            ranked = [(s, r) for s, r in ranked if s >= args.min_score]
+        ranked = ranked[: args.k]
+
+        if not ranked:
+            n_dropped_score += 1
+            continue
+
+        user_turn = reconstruct_user_turn(list(row["prompt"]))
+        tools = list(row.get("tools_orig", []) or [])
+        image_paths = list(row.get("image_paths", []) or [])
+        images = [{"bytes": read_image_bytes(p)} for p in image_paths]
+
+        for score, resp in ranked:
+            content_text, tool_calls = parse_tool_calls(resp)
+            if not tool_calls:
+                n_dropped_parse += 1
+                continue
+            assistant_turn = {
+                "role": "assistant",
+                "content": content_text or "",
+                "tool_calls": tool_calls,
+            }
+            out_records.append(
+                {
+                    "messages": [user_turn, assistant_turn],
+                    "images": images,
+                    "tools": tools,
+                }
+            )
+
+    print(f"kept {len(out_records)} sft examples; "
+          f"dropped {n_dropped_parse} (parse), {n_dropped_score} (score filter)")
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    pd.DataFrame(out_records).to_parquet(out_path, index=False)
+    print(f"wrote {out_path}")
+
+
+if __name__ == "__main__":
+    main()

data.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13d6c907cdce09739e63e32f40977e608f6afe6169b85bb58938b9df2616b3f8
+size 391661196

pip_list.txt

@@ -0,0 +1,206 @@
+absl-py                            2.4.0
+accelerate                         1.12.0
+aiohappyeyeballs                   2.6.1
+aiohttp                            3.13.3
+aiohttp-cors                       0.8.1
+aiosignal                          1.4.0
+annotated-doc                      0.0.4
+annotated-types                    0.7.0
+antlr4-python3-runtime             4.9.3
+anyio                              4.12.1
+astor                              0.8.1
+async-timeout                      5.0.1
+attrs                              25.4.0
+av                                 16.1.0
+blake3                             1.0.8
+cachetools                         6.2.5
+cbor2                              5.8.0
+certifi                            2026.1.4
+cffi                               2.0.0
+charset-normalizer                 3.4.4
+click                              8.3.1
+cloudpickle                        3.1.2
+codetiming                         1.4.0
+colorful                           0.5.8
+compressed-tensors                 0.11.0
+cupy-cuda12x                       13.6.0
+datasets                           4.5.0
+depyf                              0.19.0
+dill                               0.4.0
+diskcache                          5.6.3
+distlib                            0.4.0
+distro                             1.9.0
+dnspython                          2.8.0
+einops                             0.8.2
+email-validator                    2.3.0
+exceptiongroup                     1.3.1
+fastapi                            0.128.0
+fastapi-cli                        0.0.20
+fastapi-cloud-cli                  0.11.0
+fastar                             0.8.0
+fastrlock                          0.8.3
+filelock                           3.20.3
+flash_attn                         2.8.3+cu12torch28cxx11abitrue
+frozendict                         2.4.7
+frozenlist                         1.8.0
+fsspec                             2025.10.0
+gguf                               0.17.1
+gitdb                              4.0.12
+GitPython                          3.1.46
+google-api-core                    2.29.0
+google-auth                        2.47.0
+googleapis-common-protos           1.72.0
+grpcio                             1.76.0
+h11                                0.16.0
+hf-xet                             1.2.0
+httpcore                           1.0.9
+httptools                          0.7.1
+httpx                              0.28.1
+huggingface-hub                    0.36.0
+idna                               3.11
+importlib_metadata                 8.7.1
+interegular                        0.3.3
+Jinja2                             3.1.6
+jiter                              0.12.0
+jsonschema                         4.26.0
+jsonschema-specifications          2025.9.1
+lark                               1.2.2
+liger_kernel                       0.6.4
+llguidance                         0.7.30
+llvmlite                           0.44.0
+lm-format-enforcer                 0.11.3
+Markdown                           3.10.1
+markdown-it-py                     4.0.0
+MarkupSafe                         3.0.3
+mathruler                          0.1.0
+mdurl                              0.1.2
+mistral_common                     1.8.8
+mpmath                             1.3.0
+msgpack                            1.1.2
+msgspec                            0.20.0
+multidict                          6.7.1
+multiprocess                       0.70.18
+networkx                           3.4.2
+ninja                              1.13.0
+numba                              0.61.2
+numpy                              2.2.6
+nvidia-cublas-cu12                 12.6.4.1
+nvidia-cuda-cupti-cu12             12.6.80
+nvidia-cuda-nvrtc-cu12             12.6.77
+nvidia-cuda-runtime-cu12           12.6.77
+nvidia-cudnn-cu12                  9.10.2.21
+nvidia-cufft-cu12                  11.3.0.4
+nvidia-cufile-cu12                 1.11.1.6
+nvidia-curand-cu12                 10.3.7.77
+nvidia-cusolver-cu12               11.7.1.2
+nvidia-cusparse-cu12               12.5.4.2
+nvidia-cusparselt-cu12             0.7.1
+nvidia-nccl-cu12                   2.27.3
+nvidia-nvjitlink-cu12              12.6.85
+nvidia-nvtx-cu12                   12.6.77
+omegaconf                          2.3.0
+openai                             2.15.0
+openai-harmony                     0.0.8
+opencensus                         0.11.4
+opencensus-context                 0.1.3
+opencv-python-headless             4.13.0.90
+opentelemetry-api                  1.39.1
+opentelemetry-exporter-prometheus  0.60b1
+opentelemetry-proto                1.39.1
+opentelemetry-sdk                  1.39.1
+opentelemetry-semantic-conventions 0.60b1
+orjson                             3.11.5
+outlines_core                      0.2.11
+packaging                          25.0
+pandas                             2.3.3
+partial-json-parser                0.2.1.1.post7
+peft                               0.18.1
+pillow                             12.1.0
+pip                                25.3
+platformdirs                       4.5.1
+prometheus_client                  0.24.1
+prometheus-fastapi-instrumentator  7.1.0
+propcache                          0.4.1
+proto-plus                         1.27.0
+protobuf                           6.33.4
+psutil                             7.2.1
+py-cpuinfo                         9.0.0
+py-spy                             0.4.1
+pyarrow                            23.0.0
+pyasn1                             0.6.2
+pyasn1_modules                     0.4.2
+pybase64                           1.4.3
+pycountry                          24.6.1
+pycparser                          3.0
+pydantic                           2.12.5
+pydantic_core                      2.41.5
+pydantic-extra-types               2.11.0
+pydantic-settings                  2.12.0
+Pygments                           2.19.2
+pylatexenc                         2.10
+python-dateutil                    2.9.0.post0
+python-dotenv                      1.2.1
+python-json-logger                 4.0.0
+python-multipart                   0.0.22
+pytz                               2025.2
+pyvers                             0.1.0
+PyYAML                             6.0.3
+pyzmq                              27.1.0
+qwen-vl-utils                      0.0.14
+ray                                2.53.0
+referencing                        0.37.0
+regex                              2026.1.15
+requests                           2.32.5
+rich                               14.3.1
+rich-toolkit                       0.17.1
+rignore                            0.7.6
+rpds-py                            0.30.0
+rsa                                4.9.1
+safetensors                        0.7.0
+scipy                              1.15.3
+sentencepiece                      0.2.1
+sentry-sdk                         2.50.0
+setproctitle                       1.3.7
+setuptools                         80.9.0
+shellingham                        1.5.4
+six                                1.17.0
+smart_open                         7.5.0
+smmap                              5.0.2
+sniffio                            1.3.1
+soundfile                          0.13.1
+soxr                               1.0.0
+starlette                          0.50.0
+sympy                              1.14.0
+tensorboard                        2.20.0
+tensorboard-data-server            0.7.2
+tensordict                         0.10.0
+tiktoken                           0.12.0
+tokenizers                         0.22.2
+tomli                              2.4.0
+torch                              2.8.0+cu126
+torchaudio                         2.8.0+cu126
+torchdata                          0.11.0
+torchvision                        0.23.0+cu126
+tqdm                               4.67.1
+transformers                       4.57.0
+triton                             3.4.0
+typer                              0.21.1
+typing_extensions                  4.15.0
+typing-inspection                  0.4.2
+tzdata                             2025.3
+urllib3                            2.6.3
+uvicorn                            0.40.0
+uvloop                             0.22.1
+virtualenv                         20.36.1
+vllm                               0.11.0
+wandb                              0.24.0
+watchfiles                         1.1.1
+websockets                         16.0
+Werkzeug                           3.1.5
+wheel                              0.45.1
+wrapt                              2.0.1
+xformers                           0.0.32.post1
+xgrammar                           0.1.25
+xxhash                             3.6.0
+yarl                               1.22.0
+zipp                               3.23.0

requirements.txt

@@ -0,0 +1,132 @@
+# Generated from pip_list.txt — exact version pinning
+# Note:
+#   - torch / torchaudio / torchvision carry the +cu124 local tag;
+#     install with: pip install --index-url https://download.pytorch.org/whl/cu124 ...
+#   - conda / mamba / libmambapy / menuinst / libarchive-c are normally managed
+#     by conda, not pip. Remove them if installing into a non-conda env.
+#   - pip / setuptools / wheel are bootstrap packages; pin only if you intend to
+#     control their versions explicitly.
+
+absl-py==2.4.0
+annotated-doc==0.0.4
+anyio==4.13.0
+archspec==0.2.3
+asttokens==2.4.1
+astunparse==1.6.3
+attrs==24.2.0
+beautifulsoup4==4.12.3
+blessed==1.33.0
+boltons==24.0.0
+Brotli==1.1.0
+certifi==2024.8.30
+cffi==1.17.1
+chardet==5.2.0
+charset-normalizer==3.4.0
+click==8.3.3
+colorama==0.4.6
+cycler==0.12.1
+decorator==5.1.1
+distro==1.9.0
+dnspython==2.7.0
+exceptiongroup==1.2.2
+executing==2.1.0
+expecttest==0.2.1
+filelock==3.16.1
+fonttools==4.62.1
+frozendict==2.4.6
+fsspec==2024.10.0
+gpustat==1.1.1
+grpcio==1.80.0
+h11==0.16.0
+h2==4.1.0
+hf-xet==1.5.0
+hpack==4.0.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==1.14.0
+hyperframe==6.0.1
+hypothesis==6.115.5
+idna==3.10
+importlib-resources==6.4.5
+ipython==8.29.0
+jedi==0.19.1
+Jinja2==3.1.4
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+kiwisolver==1.5.0
+libarchive-c==5.1
+lief==0.14.1
+lintrunner==0.12.5
+Markdown==3.10.2
+markdown-it-py==4.2.0
+MarkupSafe==3.0.2
+matplotlib==3.10.9
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+modelscope==1.35.2
+more-itertools==10.5.0
+mpmath==1.3.0
+networkx==3.4.2
+ninja==1.11.1.1
+numpy<2
+nvitop==1.6.2
+optree==0.13.0
+packaging==24.1
+parso==0.8.4
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==10.2.0
+pip==24.2
+pkginfo==1.11.2
+pkgutil-resolve-name==1.3.10
+platformdirs==4.3.6
+pluggy==1.5.0
+prompt-toolkit==3.0.48
+protobuf==7.34.1
+psutil==6.1.0
+ptyprocess==0.7.0
+pure-eval==0.2.3
+pyarrow==24.0.0
+pycosat==0.6.6
+pycparser==2.22
+Pygments==2.18.0
+pyparsing==3.3.2
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+python-etcd==0.4.5
+pytz==2024.2
+PyYAML==6.0.2
+referencing==0.35.1
+requests==2.32.3
+rich==15.0.0
+rpds-py==0.20.0
+ruamel.yaml==0.18.6
+ruamel.yaml.clib==0.2.8
+setuptools==72.1.0
+shellingham==1.5.4
+six==1.16.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+stack-data==0.6.2
+sympy==1.13.1
+tensorboard==2.20.0
+tensorboard-data-server==0.7.2
+torch==2.5.1+cu124
+torchaudio==2.5.1+cu124
+torchelastic==0.2.2
+torchvision==0.20.1+cu124
+tqdm==4.66.5
+traitlets==5.14.3
+triton==3.1.0
+truststore==0.9.2
+typer==0.25.1
+types-dataclasses==0.6.6
+typing-extensions==4.12.2
+urllib3==2.2.3
+wcwidth==0.6.0
+Werkzeug==3.1.8
+wheel==0.44.0
+zipp==3.20.2
+zstandard==0.23.0

verl.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee07dc56371e123a4f5afcd399a6b1b4624ce8af1edac0f8512ef1d98f5bced9
+size 5809181481

verl/.agent/skills/issue/SKILL.md

@@ -0,0 +1,47 @@
+---
+name: issue
+description: Create or update a GitHub issue following verl project conventions.
+user_invocable: true
+---
+
+When the user asks to create or update an issue, follow these steps:
+
+### 1. Gather Context
+
+Read the following to understand available issue types and their required fields:
+
+- [`bug-report.yml`](.github/ISSUE_TEMPLATE/bug-report.yml)
+- [`feature-request.yml`](.github/ISSUE_TEMPLATE/feature-request.yml)
+
+If updating an existing issue, read its current title, body, labels, and comments first.
+
+### 2. Determine Issue Type
+
+Based on the user's description, select the appropriate template:
+
+- **Bug report** ([`bug-report.yml`](.github/ISSUE_TEMPLATE/bug-report.yml)) — something is broken or behaves unexpectedly
+- **Feature request** ([`feature-request.yml`](.github/ISSUE_TEMPLATE/feature-request.yml)) — a new capability or enhancement
+- **Blank issue** — if neither template fits
+
+### 3. Compose the Issue
+
+Fill in the template fields based on information from the user and the codebase. For bug reports, run `python scripts/diagnose.py` to gather system info if possible.
+
+When updating, ensure the title and body still accurately reflect the current state of the issue.
+
+### 4. Check for Duplicates
+
+Search for existing issues before creating:
+
+```
+gh issue list --repo verl-project/verl --state open --search "<keywords>"
+```
+
+If a duplicate exists, inform the user instead of creating a new one.
+
+### 5. Create or Update the Issue
+
+- **Create**: add `good first issue` and/or `call for contribution` labels if the issue is straightforward and suitable for new contributors.
+- **Update**: update title, body, and labels as needed.
+
+Return the issue URL when done.

verl/.agent/skills/pr/SKILL.md

@@ -0,0 +1,33 @@
+---
+name: pr
+description: Create or update a pull request following verl project conventions.
+user_invocable: true
+---
+
+When the user asks to create or update a PR, follow these steps:
+
+### 1. Gather Context
+
+Read the following and understand the current branch's changes compared to main:
+
+- [`CONTRIBUTING.md`](CONTRIBUTING.md)
+- [`PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md)
+
+If a PR already exists for this branch, also read its current title, body, and review comments.
+
+### 2. Compose PR Title and Body
+
+Follow the PR template strictly for both title format and body sections. Only check checklist boxes for steps that have actually been completed.
+
+When updating, ensure the title and body still accurately reflect **all** changes on the branch, not just the latest commit.
+
+### 3. Pre-submit Checks
+
+Run pre-commit and fix any issues before creating or pushing.
+
+### 4. Create or Update the PR
+
+- **Create**: target `main` by default unless the user specifies otherwise.
+- **Update**: push new commits and update the title and body if the scope has changed. **Read the current PR title and body first** and incorporate any edits the user may have made directly on GitHub — never overwrite with a version generated from scratch.
+
+Return the PR URL when done.

verl/.gemini/config.yaml

@@ -0,0 +1,10 @@
+have_fun: false
+code_review:
+  disable: false
+  comment_severity_threshold: HIGH
+  max_review_comments: -1
+  pull_request_opened:
+    help: false
+    summary: false
+    code_review: true
+ignore_patterns: []

verl/.git-blame-ignore-revs

@@ -0,0 +1,13 @@
+# Local uasge: git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# [dev] feat: immigrate from yapf & pylint to ruff based on pre-commit
+# Changed 268 files, +10k/-9k lines. This is the biggest formatter change.
+b00f77d8559b48d57a33c0132a5ba1c81891a536
+
+# [ci] refactor: reduce ruff line-length from 300 to 120
+# Changed 238 files, +6k/-1k lines. Global formatting change.
+00a10a8ef389556f957a2f36132b2358fd6a109f
+
+# [Lint] fix: linting errors in all files
+# Changed 179 files, +1k/-3k lines. Global lint fix.
+8e5ad4688a13de81727c014a3c2e2fb26324bc20

verl/.github/CODEOWNERS

@@ -0,0 +1,25 @@
+/docs @eric-haibin-lin @zhaochenyang20 @hongpeng-guo
+/docs/amd_tutorial @yushengsu-thu
+/docs/slang_multiturn @zhaochenyang20 @SwordFaith
+/docs/ascend_tutorial @wucong25
+
+/third_party/sglang @zhaochenyang20 @SwordFaith
+/third_party/vllm @PeterSH6 @wuxibin89
+
+/examples/grpo_trainer @vermouth1992 @PeterSH6 @tardis-key @wucong25 @ji-huazhong
+
+/verl/single_controller @zw0610 @wuxibin89 @hongpeng-guo
+/verl/trainer @eric-haibin-lin @vermouth1992 @tongyx361 @PeterSH6
+/verl/models/mcore @ISEEKYAN @vermouth1992
+/verl/models/transformers @vermouth1992 @PeterSH6 @tardis-key @wucong25 @ji-huazhong
+/verl/workers/engine @eric-haibin-lin @vermouth1992 @ZihengJiang @ArronHZG
+/verl/workers/roles @eric-haibin-lin @vermouth1992 @ZihengJiang
+/verl/workers/engine/fsdp @eric-haibin-lin @vermouth1992 @ZihengJiang
+/verl/workers/rollout/vllm_rollout @wuxibin89 @PeterSH6 @chenhaiq @ArronHZG
+/verl/workers/rollout/sglang_rollout @zhaochenyang20 @SwordFaith @chenhaiq @ArronHZG
+/verl/workers/engine/megatron @ISEEKYAN @vermouth1992
+/verl/experimental @wuxibin89 @ArronHZG
+
+/tests/single_controller @zw0610 @wuxibin89
+/tests/trainer @eric-haibin-lin @vermouth1992 @tongyx361 @PeterSH6
+/tests/workers/rollout/vllm_rollout @wuxibin89 @PeterSH6 @chenhaiq

verl/.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,65 @@
+# modified from https://github.com/huggingface/transformers/blob/main/.github/ISSUE_TEMPLATE/bug-report.yml?plain=1
+name: "\U0001F41B Bug Report"
+description: Submit a bug report to help us improve verl
+labels: [ "bug" ]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report! 🤗
+
+  - type: textarea
+    id: system-info
+    attributes:
+      label: System Info
+      description: Please share your system info with us. You can run the command `python scripts/diagnose.py` and copy-paste its output below.
+      placeholder: verl version, platform, python version, ...
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: information-scripts-examples
+    attributes:
+      label: Information
+      description: 'The problem arises when using:'
+      options:
+        - label: "The official example scripts"
+        - label: "My own modified scripts"
+
+  - type: checkboxes
+    id: information-tasks
+    attributes:
+      label: Tasks
+      description: "The tasks I am working on are:"
+      options:
+        - label: "An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)"
+        - label: "My own task or dataset (give details below)"
+
+  - type: textarea
+    id: reproduction
+    validations:
+      required: true
+    attributes:
+      label: Reproduction
+      description: |
+        Please provide a code sample that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
+        Please include relevant config information with your code.
+        If you have code snippets, error messages, stack traces please provide them here as well.
+        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
+        Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
+
+      placeholder: |
+        Steps to reproduce the behavior:
+
+          1.
+          2.
+          3.
+
+
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: Expected behavior
+      description: "A clear and concise description of what you would expect to happen."

verl/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+blank_issues_enabled: true
+version: 0.1

verl/.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,32 @@
+# modified from https://github.com/huggingface/transformers/blob/main/.github/ISSUE_TEMPLATE/feature-request.yml?plain=1
+name: "\U0001F680 Feature request"
+description: Submit a proposal/request for a new verl feature
+labels: [ "Feature request" ]
+body:
+  - type: textarea
+    id: feature-request
+    validations:
+      required: true
+    attributes:
+      label: Feature request
+      description: |
+        A clear and concise description of the feature proposal. Please provide a link to the paper and code in case they exist.
+
+  - type: textarea
+    id: motivation
+    validations:
+      required: true
+    attributes:
+      label: Motivation
+      description: |
+        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.
+
+
+  - type: textarea
+    id: contribution
+    validations:
+      required: true
+    attributes:
+      label: Your contribution
+      description: |
+        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/verl-project/verl/blob/main/CONTRIBUTING.md)

verl/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,41 @@
+### What does this PR do?
+
+> Add **concise** overview of what this PR aims to achieve or accomplish. Reference related GitHub issues and PRs that help with the review.
+
+### Checklist Before Starting
+
+- [ ] Search for similar PRs. Paste at least one query link here: ...
+- [ ] Format the PR title as `[{modules}] {type}: {description}` (This will be checked by the CI)
+  - `{modules}` include `fsdp`, `megatron`, `veomni`, `sglang`, `vllm`, `rollout`, `trainer`, `ci`, `training_utils`, `recipe`, `hardware`, `deployment`, `ray`, `worker`, `single_controller`, `misc`, `perf`, `model`, `algo`, `env`, `tool`, `ckpt`, `doc`, `data`, `cfg`, `reward`, `fully_async`, `one_step_off`
+  - If this PR involves multiple modules, separate them with `,` like `[megatron, fsdp, doc]`
+  - `{type}` is in `feat`, `fix`, `refactor`, `chore`, `test`
+  - If this PR breaks any API (CLI arguments, config, function signature, etc.), add `[BREAKING]` to the beginning of the title.
+  - Example: `[BREAKING][fsdp, megatron] feat: dynamic batching`
+
+### Test
+
+> For changes that can not be tested by CI (e.g., algorithm implementation, new model support), validate by experiment(s) and show results like training curve plots, evaluation results, etc.
+
+### API and Usage Example
+
+> Demonstrate how the API changes if any, and provide usage example(s) if possible.
+
+```python
+# Add code snippet or script demonstrating how to use this
+```
+
+### Design & Code Changes
+
+> Demonstrate the high-level design if this PR is complex, and list the specific changes.
+
+### Checklist Before Submitting
+
+> [!IMPORTANT]
+> Please check all the following items before requesting a review, otherwise the reviewer might deprioritize this PR for review.
+
+- [ ] Read the [Contribute Guide](https://github.com/verl-project/verl/blob/main/CONTRIBUTING.md).
+- [ ] Apply [pre-commit checks](https://github.com/verl-project/verl/blob/main/CONTRIBUTING.md#code-linting-and-formatting): `pre-commit install && pre-commit run --all-files --show-diff-on-failure --color=always`
+- [ ] Add / Update [the documentation](https://github.com/verl-project/verl/tree/main/docs).
+- [ ] Add unit or end-to-end test(s) to [the CI workflow](https://github.com/verl-project/verl/tree/main/.github/workflows) to cover all the code. If not feasible, explain why: ...
+- [ ] Once your PR is ready for CI, send a message in [the `ci-request` channel](https://verl-project.slack.com/archives/C091TCESWB1) in [the `verl` Slack workspace](https://join.slack.com/t/verl-project/shared_invite/zt-3855yhg8g-CTkqXu~hKojPCmo7k_yXTQ). (If not accessible, please try [the Feishu group (飞书群)](https://applink.larkoffice.com/client/chat/chatter/add_by_link?link_token=772jd4f1-cd91-441e-a820-498c6614126a).)
+- [ ] If your PR is related to the `recipe` submodule, please also update the reference to the submodule commit via `git submodule update --remote` or `cd recipe && git pull origin main`.

verl/.github/dependabot.yml

@@ -0,0 +1,9 @@
+## Enabled the dependabot to check the dependencies of the project
+## Dependabot will open pull requests to update dependencies automatically
+
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly

verl/.github/workflows/README.md

@@ -0,0 +1,73 @@
+### Adding a New Workflow
+
+When adding a new workflow for continuous integration (CI), you have two runner options: a fixed runner or a machine from the vemlp.
+
+- **Fixed Runner**: To use a fixed runner, specify it in your workflow using the `runs-on` keyword, like `runs-on: [L20x8]`. 
+- **Vemlp Runner**: Opting for a Vemlp machine allows you to launch tasks elastically. 
+
+Here is a template to assist you. This template is designed for using Vemlp machines. Currently, for each workflow, you need to create a `setup` and a `cleanup` job. When using this template, the main parts you need to modify are the `IMAGE` environment variable and the specific `job steps`.
+
+```yaml
+name: Your Default Workflow
+
+on:
+  push:
+    branches:
+      - main
+      - v0.*
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - ".github/workflows/template.yml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+permissions:
+  contents: read
+
+env:
+  IMAGE: "your vemlp image" # e.g. "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2"
+  DYNAMIC_RUNNER_URL: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" # public veFaas api
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      task-id: ${{ steps.create-runner.outputs.task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1 
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_URL }}"
+          image: "${{ env.DEFAULT_IMAGE }}"
+
+  your_job:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'default-runner' }}"]
+    steps:
+      xxxx # your jobs
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: [setup, your_job]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_URL }}"
+          task-id: "${{ needs.setup.outputs.task-id }}"
+```
+
+### Model and Dataset
+To avoid CI relies on network, we pre-download dataset on a NFS on the CI machine. The path for models are \${HOME}/models and the path for dataset is \${HOME}/models/hf_data.

verl/.github/workflows/check-pr-title.yml

@@ -0,0 +1,58 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests 
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+
+on:
+  pull_request:
+    types: [opened, edited, synchronize]
+
+jobs:
+  check-title:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Run PR title checker
+        run: python3 tests/special_sanity/check_pr_title.py
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+
+      - name: Run PR description checker
+        run: python3 tests/special_sanity/check_pr_description.py
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+          GITHUB_EVENT_PATH: ${{ github.event_path }}

verl/.github/workflows/cpu_unit_tests.yml

@@ -0,0 +1,117 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: cpu_unit_tests
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+      - v0.*
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!tests/special_sanity/**"
+      - .github/workflows/cpu_unit_tests.yml
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+env:
+  IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm018.dev1"
+  DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-image: "${{ env.IMAGE }}"
+
+  cpu_unit_tests:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 20 # Increase this timeout value as needed
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      TORCH_COMPILE_DISABLE: 1
+      TORCHINDUCTOR_DISABLE: 1
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+      - name: Download datasets
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+          python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k
+      - name: Running CPU unit tests
+        run: |
+          echo '[pytest]' > pytest.ini
+          echo 'python_files = *_on_cpu.py' >> pytest.ini
+          pytest -s -x --asyncio-mode=auto tests/
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: [setup, cpu_unit_tests]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}"

verl/.github/workflows/doc.yml

@@ -0,0 +1,101 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests 
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+
+name: doc_test
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+      - v0.*
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "docs/**"
+      - .github/workflows/doc.yml
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read      # for checkout
+  pages: write        # for deploy-pages
+  id-token: write     # for deploy-pages
+
+jobs:
+  doc_test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5 # Increase this timeout value as needed
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip install -r docs/requirements-docs.txt
+
+      - name: Run doc make html
+        run: |
+          cd docs 
+          make clean
+          make html SPHINXOPTS="--keep-going -w _build/sphinx.log"
+          if grep -q ": ERROR:" _build/sphinx.log; then
+            echo "🚨 Sphinx doc build contained ERRORs - see _build/sphinx.log"
+            exit 1
+          fi
+          if grep -q "WARNING: document isn't included in any toctree" _build/sphinx.log; then
+            echo "🚨 Sphinx doc build contained WARNING. Please include newly added docs in index.rst. See _build/sphinx.log for details"
+            exit 1
+          fi
+          if grep -q "WARNING: Inline emphasis" _build/sphinx.log; then
+            echo "🚨 Sphinx doc build contained WARNING. Please check inline emphasis is correct. See _build/sphinx.log for details"
+            exit 1
+          fi
+          if grep -q "WARNING: Definition list ends without a blank line" _build/sphinx.log; then
+            echo "🚨 Sphinx doc build contained WARNING. Please check if the indentation is correct. See _build/sphinx.log for details"
+            exit 1
+          fi

verl/.github/workflows/docker-build-ascend-a2-qwen3_5.yml

@@ -0,0 +1,123 @@
+name: docker-build-ascend-a2-qwen3_5
+
+on:
+  workflow_dispatch:
+
+env:
+  QUAY_REPO: quay.io/ascend/verl
+
+jobs:
+  build-push-digest:
+    name: build-${{ matrix.tag }}
+    if: ${{ github.repository_owner == 'verl-project' }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: linux/amd64
+            runner: ubuntu-latest
+            tag: amd64
+          - arch: linux/arm64
+            runner: ubuntu-22.04-arm
+            tag: arm64
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+          driver: docker-container
+          use: true
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push by digest
+        uses: docker/build-push-action@v6
+        id: build
+        with:
+          context: .
+          platforms: ${{ matrix.arch }}
+          file: ./docker/ascend/Dockerfile.ascend_8.5.2_a2_qwen3-5
+          push: true
+          # 关键：push-by-digest=true 只推送镜像层，不创建 tag
+          outputs: type=image,name=${{ env.QUAY_REPO }},push-by-digest=true,name-canonical=true,push=true
+          provenance: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.tag }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-image:
+    name: merge
+    runs-on: ubuntu-latest
+    needs: build-push-digest
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Merge and push multi-arch image
+        env:
+          IMAGE: ${{ env.QUAY_REPO }}
+        run: |
+          # 读取所有 digest 文件，构建 IMAGE@sha256:DIGEST 格式
+          DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests))
+          
+          echo "Digests: $DIGESTS"
+          
+          # 创建多架构 manifest
+          docker buildx imagetools create \
+            -t "$IMAGE:verl-8.5.2-910b-ubuntu22.04-py3.11-qwen3-5" \
+            $DIGESTS

verl/.github/workflows/docker-build-ascend-a2.yml

@@ -0,0 +1,281 @@
+name: docker-build-ascend-a2
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["main"]
+    paths:
+      - "docker/ascend/Dockerfile.ascend_8.5.0_a2"
+      - "docker/ascend/Dockerfile.ascend_8.5.0_a2_v0.7.1"
+      - ".github/workflows/docker-build-ascend-a2.yml"
+
+  release:
+    types: [published]
+  schedule:
+    - cron: "0 16 * * *"
+
+env:
+  QUAY_REPO: quay.io/ascend/verl
+
+jobs:
+  # ============================================================
+  # 第一阶段：分架构构建并推送 digest（main 镜像）
+  # ============================================================
+  build-push-digest:
+    name: build-${{ matrix.tag }}
+    if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.tag }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: linux/amd64
+            runner: ubuntu-latest
+            tag: amd64
+          - arch: linux/arm64
+            runner: ubuntu-22.04-arm
+            tag: arm64
+    outputs:
+      tag: ${{ steps.version.outputs.tag }}
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Get base image name and tag
+        id: base_image
+        run: |
+          BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend_8.5.0_a2 | head -1 | cut -d' ' -f2)
+          echo "Base image full: $BASE_IMAGE_FULL" 
+          BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2)
+          echo "Base image tag: $BASE_IMAGE_TAG"
+          NEW_IMAGE_NAME="verl-$BASE_IMAGE_TAG"
+          echo "New image name: $NEW_IMAGE_NAME"  
+          echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT"
+          echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT"
+
+      - name: Get image tag
+        id: version
+        run: |
+          BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g')
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT"
+          elif [ "$BRANCH_NAME" = "main" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+          driver: docker-container
+          use: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push by digest (main image)
+        uses: docker/build-push-action@v6
+        id: build_main
+        with:
+          context: .
+          platforms: ${{ matrix.arch }}
+          file: ./docker/ascend/Dockerfile.ascend_8.5.0_a2
+          push: true
+          outputs: type=image,name=${{ env.QUAY_REPO }},push-by-digest=true,name-canonical=true,push=true
+          provenance: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+
+      - name: Export digest (main image)
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build_main.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/main-${digest#sha256:}"
+
+      - name: Upload digest (main image)
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-main-${{ matrix.tag }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # ============================================================
+  # 第一阶段（v0.7.1）：分架构构建并推送 digest
+  # ============================================================
+  build-push-digest-v071:
+    name: build-v0.7.1-${{ matrix.tag }}
+    if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-v0.7.1-${{ matrix.tag }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: linux/amd64
+            runner: ubuntu-latest
+            tag: amd64
+          - arch: linux/arm64
+            runner: ubuntu-22.04-arm
+            tag: arm64
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+          driver: docker-container
+          use: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push by digest (v0.7.1 image)
+        uses: docker/build-push-action@v6
+        id: build_v071
+        with:
+          context: .
+          platforms: ${{ matrix.arch }}
+          file: ./docker/ascend/Dockerfile.ascend_8.5.0_a2_v0.7.1
+          push: true
+          outputs: type=image,name=${{ env.QUAY_REPO }},push-by-digest=true,name-canonical=true,push=true
+          provenance: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+
+      - name: Export digest (v0.7.1 image)
+        run: |
+          mkdir -p ${{ runner.temp }}/digests-v071
+          digest="${{ steps.build_v071.outputs.digest }}"
+          touch "${{ runner.temp }}/digests-v071/v071-${digest#sha256:}"
+
+      - name: Upload digest (v0.7.1 image)
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-v071-${{ matrix.tag }}
+          path: ${{ runner.temp }}/digests-v071/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # ============================================================
+  # 第二阶段：合并多架构 manifest（main 镜像）
+  # ============================================================
+  merge-image:
+    name: merge
+    runs-on: ubuntu-latest
+    needs: build-push-digest
+    steps:
+      - name: Download digests (main image)
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests-main
+          pattern: digests-main-*
+          merge-multiple: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Merge and push multi-arch image (main)
+        env:
+          IMAGE: ${{ env.QUAY_REPO }}
+        run: |
+          DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-main | sed 's/^main-//'))
+          echo "Main image digests: $DIGESTS"
+          docker buildx imagetools create \
+            -t "$IMAGE:${{ needs.build-push-digest.outputs.tag }}" \
+            $DIGESTS
+
+  # ============================================================
+  # 第二阶段（v0.7.1）：合并多架构 manifest
+  # ============================================================
+  merge-image-v071:
+    name: merge-v0.7.1
+    runs-on: ubuntu-latest
+    needs: build-push-digest-v071
+    steps:
+      - name: Download digests (v0.7.1 image)
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests-v071
+          pattern: digests-v071-*
+          merge-multiple: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Merge and push multi-arch image (v0.7.1)
+        env:
+          IMAGE: ${{ env.QUAY_REPO }}
+        run: |
+          DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-v071 | sed 's/^v071-//'))
+          echo "v0.7.1 image digests: $DIGESTS"
+          docker buildx imagetools create \
+            -t "$IMAGE:verl-8.5.0-910b-ubuntu22.04-py3.11-v0.7.1" \
+            $DIGESTS

verl/.github/workflows/docker-build-ascend-a3-qwen3_5.yml

@@ -0,0 +1,123 @@
+name: docker-build-ascend-a3-qwen3_5
+
+on:
+  workflow_dispatch:
+
+env:
+  QUAY_REPO: quay.io/ascend/verl
+
+jobs:
+  build-push-digest:
+    name: build-${{ matrix.tag }}
+    if: ${{ github.repository_owner == 'verl-project' }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: linux/amd64
+            runner: ubuntu-latest
+            tag: amd64
+          - arch: linux/arm64
+            runner: ubuntu-22.04-arm
+            tag: arm64
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+          driver: docker-container
+          use: true
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push by digest
+        uses: docker/build-push-action@v6
+        id: build
+        with:
+          context: .
+          platforms: ${{ matrix.arch }}
+          file: ./docker/ascend/Dockerfile.ascend_8.5.2_a3_qwen3-5
+          push: true
+          # 关键：push-by-digest=true 只推送镜像层，不创建 tag
+          outputs: type=image,name=${{ env.QUAY_REPO }},push-by-digest=true,name-canonical=true,push=true
+          provenance: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.tag }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-image:
+    name: merge
+    runs-on: ubuntu-latest
+    needs: build-push-digest
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Merge and push multi-arch image
+        env:
+          IMAGE: ${{ env.QUAY_REPO }}
+        run: |
+          # 读取所有 digest 文件，构建 IMAGE@sha256:DIGEST 格式
+          DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests))
+          
+          echo "Digests: $DIGESTS"
+          
+          # 创建多架构 manifest
+          docker buildx imagetools create \
+            -t "$IMAGE:verl-8.5.2-a3-ubuntu22.04-py3.11-qwen3-5" \
+            $DIGESTS

verl/.github/workflows/docker-build-ascend-a3.yml

@@ -0,0 +1,281 @@
+name: docker-build-ascend-a3
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["main"]
+    paths:
+      - "docker/ascend/Dockerfile.ascend_8.5.0_a3"
+      - "docker/ascend/Dockerfile.ascend_8.5.0_a3_v0.7.1"
+      - ".github/workflows/docker-build-ascend-a3.yml"
+
+  release:
+    types: [published]
+  schedule:
+    - cron: "0 19 * * *"
+
+env:
+  QUAY_REPO: quay.io/ascend/verl
+
+jobs:
+  # ============================================================
+  # 第一阶段：分架构构建并推送 digest（main 镜像）
+  # ============================================================
+  build-push-digest:
+    name: build-${{ matrix.tag }}
+    if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.tag }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: linux/amd64
+            runner: ubuntu-latest
+            tag: amd64
+          - arch: linux/arm64
+            runner: ubuntu-22.04-arm
+            tag: arm64
+    outputs:
+      tag: ${{ steps.version.outputs.tag }}
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Get base image name and tag
+        id: base_image
+        run: |
+          BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend_8.5.0_a3 | head -1 | cut -d' ' -f2)
+          echo "Base image full: $BASE_IMAGE_FULL" 
+          BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2)
+          echo "Base image tag: $BASE_IMAGE_TAG"
+          NEW_IMAGE_NAME="verl-$BASE_IMAGE_TAG"
+          echo "New image name: $NEW_IMAGE_NAME"  
+          echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT"
+          echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT"
+
+      - name: Get image tag
+        id: version
+        run: |
+          BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g')
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT"
+          elif [ "$BRANCH_NAME" = "main" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+          driver: docker-container
+          use: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push by digest (main image)
+        uses: docker/build-push-action@v6
+        id: build_main
+        with:
+          context: .
+          platforms: ${{ matrix.arch }}
+          file: ./docker/ascend/Dockerfile.ascend_8.5.0_a3
+          push: true
+          outputs: type=image,name=${{ env.QUAY_REPO }},push-by-digest=true,name-canonical=true,push=true
+          provenance: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+
+      - name: Export digest (main image)
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build_main.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/main-${digest#sha256:}"
+
+      - name: Upload digest (main image)
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-main-${{ matrix.tag }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # ============================================================
+  # 第一阶段（v0.7.1）：分架构构建并推送 digest
+  # ============================================================
+  build-push-digest-v071:
+    name: build-v0.7.1-${{ matrix.tag }}
+    if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-v0.7.1-${{ matrix.tag }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: linux/amd64
+            runner: ubuntu-latest
+            tag: amd64
+          - arch: linux/arm64
+            runner: ubuntu-22.04-arm
+            tag: arm64
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+          driver: docker-container
+          use: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push by digest (v0.7.1 image)
+        uses: docker/build-push-action@v6
+        id: build_v071
+        with:
+          context: .
+          platforms: ${{ matrix.arch }}
+          file: ./docker/ascend/Dockerfile.ascend_8.5.0_a3_v0.7.1
+          push: true
+          outputs: type=image,name=${{ env.QUAY_REPO }},push-by-digest=true,name-canonical=true,push=true
+          provenance: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+
+      - name: Export digest (v0.7.1 image)
+        run: |
+          mkdir -p ${{ runner.temp }}/digests-v071
+          digest="${{ steps.build_v071.outputs.digest }}"
+          touch "${{ runner.temp }}/digests-v071/v071-${digest#sha256:}"
+
+      - name: Upload digest (v0.7.1 image)
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-v071-${{ matrix.tag }}
+          path: ${{ runner.temp }}/digests-v071/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # ============================================================
+  # 第二阶段：合并多架构 manifest（main 镜像）
+  # ============================================================
+  merge-image:
+    name: merge
+    runs-on: ubuntu-latest
+    needs: build-push-digest
+    steps:
+      - name: Download digests (main image)
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests-main
+          pattern: digests-main-*
+          merge-multiple: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Merge and push multi-arch image (main)
+        env:
+          IMAGE: ${{ env.QUAY_REPO }}
+        run: |
+          DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-main | sed 's/^main-//'))
+          echo "Main image digests: $DIGESTS"
+          docker buildx imagetools create \
+            -t "$IMAGE:${{ needs.build-push-digest.outputs.tag }}" \
+            $DIGESTS
+
+  # ============================================================
+  # 第二阶段（v0.7.1）：合并多架构 manifest
+  # ============================================================
+  merge-image-v071:
+    name: merge-v0.7.1
+    runs-on: ubuntu-latest
+    needs: build-push-digest-v071
+    steps:
+      - name: Download digests (v0.7.1 image)
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests-v071
+          pattern: digests-v071-*
+          merge-multiple: true
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Merge and push multi-arch image (v0.7.1)
+        env:
+          IMAGE: ${{ env.QUAY_REPO }}
+        run: |
+          DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-v071 | sed 's/^v071-//'))
+          echo "v0.7.1 image digests: $DIGESTS"
+          docker buildx imagetools create \
+            -t "$IMAGE:verl-8.5.0-a3-ubuntu22.04-py3.11-v0.7.1" \
+            $DIGESTS

verl/.github/workflows/docker-build-ascend-sglang-a2.yml

@@ -0,0 +1,84 @@
+name: docker-build-ascend-sglang-a2
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["main"]
+    paths:
+      - "docker/ascend/Dockerfile.ascend.sglang_8.5.0_a2"
+      - ".github/workflows/docker-build-ascend-sglang-a2.yml"
+  release:
+    types: [published]
+  schedule:
+    - cron: "0 16 * * *"
+
+jobs:
+  build-ascend-sglang-image-a2:
+    if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }}
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-build-ascend-sglang-image-a2
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Get base image name and tag
+        id: base_image
+        run: |
+          BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend.sglang_8.5.0_a2 | head -1 | cut -d' ' -f2)
+          echo "Base image full: $BASE_IMAGE_FULL" 
+          BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2)
+          echo "Base image tag: $BASE_IMAGE_TAG"
+          NEW_IMAGE_NAME="verl-sglang-$BASE_IMAGE_TAG"
+          echo "New image name: $NEW_IMAGE_NAME"  
+          echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT"
+          echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT"
+
+      - name: Get image tag
+        id: version
+        run: |
+          BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g')
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT"
+          elif [ "$BRANCH_NAME" = "main" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push images Quay
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./docker/ascend/Dockerfile.ascend.sglang_8.5.0_a2
+          push: true
+          tags: |
+            quay.io/ascend/verl:${{ steps.version.outputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1

verl/.github/workflows/docker-build-ascend-sglang-a3.yml

@@ -0,0 +1,84 @@
+name: docker-build-ascend-sglang-a3
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["main"]
+    paths:
+      - "docker/ascend/Dockerfile.ascend.sglang_8.5.0_a3"
+      - ".github/workflows/docker-build-ascend-sglang-a3.yml"
+  release:
+    types: [published]
+  schedule:
+    - cron: "0 16 * * *"
+
+jobs:
+  build-ascend-sglang-image-a3:
+    if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }}
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-build-ascend-sglang-image-a3
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+    steps:
+      - name: Remove unnecessary parts in github actions runners to free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Get base image name and tag
+        id: base_image
+        run: |
+          BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend.sglang_8.5.0_a3 | head -1 | cut -d' ' -f2)
+          echo "Base image full: $BASE_IMAGE_FULL" 
+          BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2)
+          echo "Base image tag: $BASE_IMAGE_TAG"
+          NEW_IMAGE_NAME="verl-sglang-$BASE_IMAGE_TAG"
+          echo "New image name: $NEW_IMAGE_NAME"  
+          echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT"
+          echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT"
+
+      - name: Get image tag
+        id: version
+        run: |
+          BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g')
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT"
+          elif [ "$BRANCH_NAME" = "main" ]; then
+            echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Clean Docker cache before build
+        run: |
+          docker system prune -a -f --volumes || true
+
+      - name: Build and push images Quay
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./docker/ascend/Dockerfile.ascend.sglang_8.5.0_a3
+          push: true
+          tags: |
+            quay.io/ascend/verl:${{ steps.version.outputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1

verl/.github/workflows/e2e_ascend.yml

@@ -0,0 +1,215 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_ascend
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+      - v0.*
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/e2e_ascend.yml"
+      - "examples/data_preprocess/**"
+      - "examples/grpo_trainer/**"
+      - "examples/ppo_trainer/**"
+      - "examples/sft/**"
+      - "verl/experimental/one_step_off_policy/**"
+      - "tests/special_npu/**"
+      - "tests/special_sanity/check_device_api_usage.py"
+      - "verl/**"
+      - "pyproject.toml"
+      - "requirements-npu.txt"
+      - "setup.py"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+permissions:
+  contents: read
+
+jobs:
+  llm_rl_job:
+    if: github.repository_owner == 'verl-project'
+    name: E2E Ascend testing for RL training scenarios of LLM models
+    runs-on: linux-aarch64-a3-8
+    timeout-minutes: 120
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-a3-ubuntu22.04-py3.11-latest
+      options: >-
+        --shm-size 16g
+    env:
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+          npu-smi info
+      - name: Check initial pip list from image
+        run: |
+          pip list
+      - name: Checkout verl-project/verl repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: true
+      - name: Install the current repository
+        run: |
+          pip install --no-deps -e .
+      - name: Check final pip list
+        run: |
+          pip list
+      - name: Preprocess gsm8k dataset
+        run: |
+          python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
+      - name: Running gsm8k e2e training tests with PPO on ASCEND NPU (FSDP backend)
+        run: |
+          ray stop --force
+          bash tests/special_npu/run_qwen3_06b_ppo.sh
+          rm -rf $HOME/ckpts
+      - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (FSDP backend)
+        run: |
+          ray stop --force
+          bash tests/special_npu/run_qwen2_5_05b_grpo.sh
+          rm -rf $HOME/ckpts
+      - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeed backend)
+        run: |
+          ray stop --force
+          USE_DIST_CKPT=True bash tests/special_npu/run_qwen2_5_05b_grpo_mindspeed.sh
+          rm -rf $HOME/dist_ckpt/qwen2_5_05b_grpo_mindspeed
+          rm -rf $HOME/ckpts
+      - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeed backend, MoE Model)
+        run: |
+          ray stop --force
+          USE_DIST_CKPT=True USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen3moe_minimal.json DUMMY_MODEL_PATH=$HOME/dist_ckpt/qwen3_30b_grpo_mindspeed bash tests/special_npu/run_qwen3_30b_grpo_mindspeed.sh
+
+  engine_mindspeed_llm_rl_job:
+    if: github.repository_owner == 'verl-project'
+    name: E2E Ascend testing for RL training scenarios of LLM models using MindSpeed_LLM engine
+    runs-on: linux-aarch64-a3-8
+    timeout-minutes: 120
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-sglang-8.3.rc1-a3-ubuntu22.04-py3.11-latest
+      options: >-
+        --shm-size 16g
+    env:
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+          npu-smi info
+      - name: Check initial pip list from image
+        run: |
+          pip list
+      - name: Checkout verl-project/verl repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: true
+      - name: Install the current repository
+        run: |
+          pip install --no-deps --no-build-isolation -e .
+      - name: Check final pip list
+        run: |
+          pip list
+      - name: Configure related dependencies
+        run: |
+          git clone --depth 1 --branch core_v0.12.1 https://github.com/NVIDIA/Megatron-LM.git /Megatron-LM
+          rm -rf /MindSpeed
+          git clone https://gitcode.com/ascend/MindSpeed.git /MindSpeed
+          git clone https://gitcode.com/ascend/MindSpeed-LLM.git /MindSpeed-LLM
+      - name: Preprocess gsm8k dataset
+        run: |
+          python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
+      - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeedLLM backend)
+        run: |
+          ray stop --force
+          export PYTHONPATH=$PYTHONPATH:/Megatron-LM
+          export PYTHONPATH=$PYTHONPATH:/MindSpeed
+          export PYTHONPATH=$PYTHONPATH:/MindSpeed-LLM
+          bash tests/special_npu/run_qwen3_8b_grpo_mindspeedllm.sh
+      - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeedLLM backend, MoE Model)
+        run: |
+          ray stop --force
+          export PYTHONPATH=$PYTHONPATH:/Megatron-LM
+          export PYTHONPATH=$PYTHONPATH:/MindSpeed
+          export PYTHONPATH=$PYTHONPATH:/MindSpeed-LLM
+          USE_DIST_CKPT=True USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen3moe_minimal.json DUMMY_MODEL_PATH=$HOME/dist_ckpt/qwen3_30b_grpo_mindspeedllm bash tests/special_npu/run_qwen3_30b_grpo_mindspeedllm.sh
+
+  vlm_rl_job:
+    if: github.repository_owner == 'verl-project'
+    name: E2E Ascend testing for RL training scenarios of VLM models
+    runs-on: linux-aarch64-a3-8
+    timeout-minutes: 120
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-a3-ubuntu22.04-py3.11-latest
+      options: >-
+        --shm-size 16g
+    env:
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+          npu-smi info
+      - name: Check initial pip list from image
+        run: |
+          pip list
+      - name: Checkout verl-project/verl repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: true
+      - name: Install the current repository
+        run: |
+          pip install --no-deps -e .
+      - name: Check final pip list
+        run: |
+          pip list
+      - name: Preprocess geo3k dataset
+        run: |
+          python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k
+      - name: Running geo3k e2e training tests with GRPO on ASCEND NPU
+        run: |
+          ray stop --force
+          bash tests/special_npu/run_qwen2_5_vl_3b_npu.sh
+          rm -rf $HOME/ckpts

verl/.github/workflows/e2e_fully_async_policy.yml

@@ -0,0 +1,169 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_fully_async_policy
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  # For push, for now only anti-patterns are specified so it is more conservative
+  # and achieves higher coverage.
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/*trainer*"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      - "verl/experimental/fully_async_policy"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      # Home
+      - "verl/experimental/fully_async_policy"
+      # Entrypoints
+      - ".github/workflows/e2e_fully_async_policy.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "tests/special_e2e/run_fully_async_policy.sh"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+env:
+  IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm018.dev1"
+  DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-image: "${{ env.IMAGE }}"
+
+  # Test FSDP2 strategy
+  e2e_fully_async_policy_fsdp2:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 10 # Increase timeout for async training
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "fsdp2"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install cupy-cuda12x==13.6.0
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+      - name: Running the E2E test with fully_async_policy algorithm (FSDP2)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_fully_async_policy.sh
+
+  # Test Megatron strategy
+  e2e_fully_async_policy_megatron:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 10 # Increase timeout for async training
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "megatron"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install cupy-cuda12x==13.6.0
+          pip3 install git+https://github.com/ISEEKYAN/mbridge.git@main --no-deps --no-build-isolation
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+      - name: Running the E2E test with fully_async_policy algorithm (Megatron)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_fully_async_policy.sh
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: [setup, e2e_fully_async_policy_fsdp2]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}"

verl/.github/workflows/e2e_fully_async_policy_ascend.yml

@@ -0,0 +1,169 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_fully_async_policy_ascend
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  # For push, for now only anti-patterns are specified so it is more conservative
+  # and achieves higher coverage.
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/*trainer*"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      - "verl/experimental/fully_async_policy"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      # Home
+      - "verl/experimental/fully_async_policy"
+      # Entrypoints
+      - ".github/workflows/e2e_fully_async_policy_ascend.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "tests/special_e2e/run_fully_async_policy.sh"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+jobs:
+  # Test FSDP2 strategy
+  e2e_fully_async_policy_fsdp2_ascend:
+    if: github.repository_owner == 'verl-project'
+    runs-on: linux-aarch64-a3-8
+    timeout-minutes: 60 # Increase this timeout value as needed
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-a3-ubuntu22.04-py3.11-latest
+      options: >-
+        --shm-size 16g
+    env:
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "fsdp2"
+      device_name: "npu"
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+          npu-smi info
+      - name: Check initial pip list from image
+        run: |
+          pip list
+      - name: Checkout verl-project/verl repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: true
+      - name: Install the current repository
+        run: |
+          pip install --no-deps -e .
+      - name: Check final pip list
+        run: |
+          pip list
+      - name: Prepare weights
+        run: |
+          ln -s /root/.cache/models ~/models
+      - name: Prepare GSM8K dataset
+        run: |
+          python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
+      - name: Running the E2E test with fully_async_policy algorithm (FSDP2)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_fully_async_policy.sh
+
+  # Test Megatron strategy
+  e2e_fully_async_policy_megatron_ascend:
+    if: github.repository_owner == 'verl-project'
+    runs-on: linux-aarch64-a3-8
+    timeout-minutes: 60 # Increase this timeout value as needed
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-a3-ubuntu22.04-py3.11-latest
+      options: >-
+        --shm-size 16g
+    env:
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "megatron"
+      device_name: "npu"
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+          npu-smi info
+      - name: Check initial pip list from image
+        run: |
+          pip list
+      - name: Checkout verl-project/verl repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: true
+      - name: Install the current repository
+        run: |
+          pip install --no-deps -e .
+      - name: Check final pip list
+        run: |
+          pip list
+      - name: Prepare weights
+        run: |
+          ln -s /root/.cache/models ~/models
+      - name: Prepare GSM8K dataset
+        run: |
+          python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
+      - name: Running the E2E test with fully_async_policy algorithm (Megatron)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_fully_async_policy.sh

verl/.github/workflows/e2e_fully_async_policy_opd.yml

@@ -0,0 +1,105 @@
+name: e2e_fully_async_policy_opd
+
+on:
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      - "!examples/*trainer*"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      - "verl/experimental/fully_async_policy"
+      - "verl/trainer/distillation"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      - "verl/experimental/fully_async_policy"
+      - "verl/trainer/distillation"
+      - ".github/workflows/e2e_fully_async_policy_opd.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "examples/data_preprocess/geo3k.py"
+      - "tests/special_e2e/run_fully_async_policy_opd.sh"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+env:
+  IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm018.dev1"
+  DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-image: "${{ env.IMAGE }}"
+
+  # Test Multi-Teacher OPD with Megatron strategy
+  e2e_fully_async_policy_opd_megatron:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 30
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install cupy-cuda12x==13.6.0
+          pip3 install --force-reinstall --no-deps --no-build-isolation git+https://github.com/ISEEKYAN/mbridge.git@main
+      - name: Prepare datasets
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+          python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ --local_save_dir ${HOME}/data/geo3k
+      - name: Running the E2E test with fully_async_policy + Multi-Teacher OPD (Megatron)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_fully_async_policy_opd.sh
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: [setup, e2e_fully_async_policy_opd_megatron]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}"

verl/.github/workflows/e2e_fully_async_policy_trtllm.yml

@@ -0,0 +1,171 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_fully_async_policy_trtllm
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      - "!examples/*trainer*"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      - "verl/workers/rollout/trtllm_rollout/**"
+      - "verl/experimental/fully_async_policy/**"
+      - "verl/checkpoint_engine/**"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "verl/workers/rollout/trtllm_rollout/**"
+      - "verl/experimental/fully_async_policy/**"
+      - "verl/checkpoint_engine/**"
+      - ".github/workflows/e2e_fully_async_policy_trtllm.yml"
+      - "tests/special_e2e/run_fully_async_policy.sh"
+      - "examples/data_preprocess/gsm8k.py"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+env:
+  IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:trtllm1.3.0rc14"
+  DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-image: "${{ env.IMAGE }}"
+
+  trtllm_async_unit_tests:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 30
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0"
+      TORCH_CUDA_ARCH_LIST: "7.5;8.0;8.9;9.0;10.0;12.0+PTX"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Reinstall FlashInfer for runtime GPU arch
+        run: |
+          unset TORCH_CUDA_ARCH_LIST
+          pip3 install --force-reinstall --no-deps flashinfer-python
+      - name: Install the current repository
+        run: |
+          pip3 install pytest-asyncio
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+      - name: Run TRT-LLM async unit tests
+        run: |
+          unset TORCH_CUDA_ARCH_LIST
+          export TRTLLM_ENABLE_PDL=0
+          export TRTLLM_TEST_MODEL_PATH_ROOT="${HOME}/models"
+          ray stop --force
+          pytest -v -s --durations=20 \
+            tests/workers/rollout/rollout_trtllm/test_trtllm_abort.py
+
+  # Megatron multi-replica: 4 rollout GPUs split into 2 replicas × TP=2.
+  e2e_fully_async_policy_trtllm_megatron_multi_replica:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 20
+    env:
+      HTTP_PROXY: ${{ secrets.PROXY_HTTP }}
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0"
+      ACTOR_STRATEGY: "megatron"
+      ROLLOUT_NAME: "trtllm"
+      GEN_TP: "2"  # 4 rollout GPUs / TP=2 = 2 replicas
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install cupy-cuda12x==14.0.1
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+      - name: Running TRT-LLM fully-async E2E test with Megatron (multi-replica, 2×TP=2)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_fully_async_policy.sh \
+            data.max_response_length=128 \
+            actor_rollout_ref.rollout.n=2 \
+            actor_rollout_ref.rollout.gpu_memory_utilization=0.1 \
+            actor_rollout_ref.actor.ppo_mini_batch_size=4 \
+            actor_rollout_ref.actor.ppo_max_token_len_per_gpu=1536
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: [setup, trtllm_async_unit_tests, e2e_fully_async_policy_trtllm_megatron_multi_replica]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}"

verl/.github/workflows/e2e_one_step_off_policy.yml

@@ -0,0 +1,169 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_one_step_off_policy
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  # For push, for now only anti-patterns are specified so it is more conservative
+  # and achieves higher coverage.
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/*trainer*"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      - "verl/experimental/one_step_off_policy"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      # Home
+      - "verl/experimental/one_step_off_policy"
+      # Entrypoints
+      - ".github/workflows/e2e_one_step_off_policy.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "tests/special_e2e/run_one_step_off_policy.sh"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+env:
+  IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm018.dev1"
+  DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-image: "${{ env.IMAGE }}"
+
+  # Test FSDP2 strategy
+  e2e_one_step_off_policy_fsdp2:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 10 # Increase timeout for async training
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "fsdp2"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install cupy-cuda12x==13.6.0
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+      - name: Running the E2E test with one_step_off_policy algorithm (FSDP2)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_one_step_off_policy.sh
+
+  # Test Megatron strategy
+  e2e_one_step_off_policy_megatron:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 10 # Increase timeout for async training
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "megatron"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install cupy-cuda12x==13.6.0
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+      - name: Running the E2E test with one_step_off_policy algorithm (Megatron)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_one_step_off_policy.sh
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs:
+      [setup, e2e_one_step_off_policy_fsdp2, e2e_one_step_off_policy_megatron]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}"

verl/.github/workflows/e2e_one_step_off_policy_ascend.yml

@@ -0,0 +1,170 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_one_step_off_policy_ascend
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  # For push, for now only anti-patterns are specified so it is more conservative
+  # and achieves higher coverage.
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/*trainer*"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      - "verl/experimental/one_step_off_policy"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      - "!**/*.md"
+      - "!**/*.sh"
+      # Other entrypoints
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      # Home
+      - "verl/experimental/one_step_off_policy"
+      # Entrypoints
+      - ".github/workflows/e2e_one_step_off_policy_ascend.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "tests/special_e2e/run_one_step_off_policy.sh"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+jobs:
+  # Test FSDP2 strategy
+  e2e_one_step_off_policy_fsdp2_ascend:
+    if: github.repository_owner == 'verl-project'
+    runs-on: linux-aarch64-a3-8
+    timeout-minutes: 60 # Increase this timeout value as needed
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-a3-ubuntu22.04-py3.11-latest
+      options: >-
+        --shm-size 16g
+    env:
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "fsdp2"
+      device_name: "npu"
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+          npu-smi info
+      - name: Check initial pip list from image
+        run: |
+          pip list
+      - name: Checkout verl-project/verl repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: true
+      - name: Install the current repository
+        run: |
+          pip install --no-deps -e .
+      - name: Check final pip list
+        run: |
+          pip list
+      - name: Prepare weights
+        run: |
+          ln -s /root/.cache/models ~/models
+      - name: Prepare GSM8K dataset
+        run: |
+          python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
+      - name: Running the E2E test with one_step_off_policy algorithm (FSDP2)
+        run: |
+          ray stop --force
+          bash tests/special_e2e/run_one_step_off_policy.sh
+
+  # Test Megatron strategy
+  e2e_one_step_off_policy_megatron_ascend:
+    if: github.repository_owner == 'verl-project'
+    runs-on: linux-aarch64-a3-8
+    timeout-minutes: 60 # Increase this timeout value as needed
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-a3-ubuntu22.04-py3.11-latest
+      options: >-
+        --shm-size 16g
+    env:
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ACTOR_STRATEGY: "megatron"
+      device_name: "npu"
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+          npu-smi info
+      - name: Check initial pip list from image
+        run: |
+          pip list
+      - name: Checkout verl-project/verl repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: true
+      - name: Install the current repository
+        run: |
+          pip install --no-deps -e .
+      - name: Check final pip list
+        run: |
+          pip list
+      - name: Prepare weights
+        run: |
+          ln -s /root/.cache/models ~/models
+      - name: Prepare GSM8K dataset
+        run: |
+          python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
+      - name: Running the E2E test with one_step_off_policy algorithm (Megatron)
+        run: |
+          ray stop --force
+          export PYTHONPATH=$PYTHONPATH:/Megatron-LM
+          bash tests/special_e2e/run_one_step_off_policy.sh

verl/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml

@@ -0,0 +1,289 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_ppo_trainer_megatron_trtllm
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch.
+  # For push, for now only anti-patterns are specified so it is more conservative
+  # and achieves higher coverage.
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      # Other entrypoints
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      # Recipes
+      - "!recipe/**"
+      # FSDP
+      - "!verl/workers/**/*dp_*.py"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      # Other entrypoints
+      - "!docker/**"
+      # Docs
+      - "!**/*.md"
+      - "!docs/**"
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      # Recipes
+      - "!recipe/**"
+      # FSDP
+      - "!verl/workers/**/*dp_*.py"
+      # Entrypoints
+      - "verl/workers/rollout/trtllm_rollout/**"
+      - "tests/workers/rollout/rollout_trtllm/**"
+      - ".github/workflows/e2e_ppo_grpo_trainer_trtllm.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "examples/data_preprocess/geo3k.py"
+      - "examples/data_preprocess/dapo_multiturn_w_tool.py"
+      - "examples/data_preprocess/aime2024_multiturn_w_tool.py"
+      - "examples/grpo_trainer/run_qwen3_8b_megatron.sh"
+      - "examples/grpo_trainer/run_qwen3_30b_a3b_megatron.sh"
+      # add back when ppo flow is ready
+      # - "tests/special_e2e/run_ppo_trainer_megatron.sh"
+      # - "verl/trainer/main_ppo.py"
+      # - "verl/trainer/config/ppo_megatron_trainer.yaml"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+env:
+  IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:trtllm1.3.0rc14"
+  DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-image: "${{ env.IMAGE }}"
+
+  trtllm_unit_tests:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 30 # Increase this timeout value as needed
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      TORCH_CUDA_ARCH_LIST: "7.5;8.0;8.9;9.0;10.0;12.0+PTX"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Reinstall FlashInfer for runtime GPU arch
+        run: |
+          unset TORCH_CUDA_ARCH_LIST
+          pip3 install --force-reinstall --no-deps flashinfer-python
+      - name: Install the current repository
+        run: |
+          pip3 install pytest-asyncio
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+      - name: Run TRTLLM unit tests
+        run: |
+          unset TORCH_CUDA_ARCH_LIST  # Let TRT-LLM CUTLASS DSL auto-detect runtime GPU arch
+          export TRTLLM_ENABLE_PDL=0  # Disable Programmatic Dependent Launch (uses SM 9.0 griddepcontrol)
+          export TRTLLM_TEST_MODEL_PATH_ROOT="${HOME}/models"
+          ray stop --force
+          pytest -v -s --durations=20 \
+            tests/workers/rollout/rollout_trtllm/test_adapter.py \
+            tests/workers/rollout/rollout_trtllm/test_async_server.py \
+            tests/workers/rollout/rollout_trtllm/test_inter_node_rollout.py \
+            tests/workers/rollout/rollout_trtllm/test_trtllm_rollout_utils.py
+
+  e2e_grpo_trainer_megatron-qwen2:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 30 # Increase this timeout value as needed
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      TORCH_CUDA_ARCH_LIST: "7.5;8.0;8.9;9.0;10.0;12.0+PTX"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Reinstall FlashInfer for runtime GPU arch
+        run: |
+          unset TORCH_CUDA_ARCH_LIST
+          pip3 install --force-reinstall --no-deps flashinfer-python
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_save_dir ${PWD}/data/gsm8k
+      - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen)
+        run: |
+          unset TORCH_CUDA_ARCH_LIST  # Let TRT-LLM CUTLASS DSL auto-detect runtime GPU arch
+          export TRTLLM_ENABLE_PDL=0  # Disable Programmatic Dependent Launch (uses SM 9.0 griddepcontrol)
+          ray stop --force
+          ACTOR_TP=2 \
+          ACTOR_PP=1 \
+          ROLLOUT_TP=2 \
+          INFER_BACKEND=trtllm \
+          MODEL_PATH="${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct" \
+            bash examples/grpo_trainer/run_qwen3_8b_megatron.sh \
+            trainer.total_training_steps=1 \
+            data.train_batch_size=32 \
+            data.max_prompt_length=128 \
+            data.max_response_length=64 \
+            actor_rollout_ref.rollout.n=1 \
+            actor_rollout_ref.actor.ppo_mini_batch_size=32 \
+            actor_rollout_ref.rollout.max_num_seqs=32 \
+            actor_rollout_ref.rollout.max_num_batched_tokens=1024 \
+            actor_rollout_ref.rollout.max_model_len=256 \
+            data.train_files="['${PWD}/data/gsm8k/train.parquet']" \
+            data.val_files="['${PWD}/data/gsm8k/test.parquet']" \
+            trainer.logger='["console"]'
+      - name: clean up
+        run: |
+          rm -rf checkpoints
+  e2e_grpo_trainer_megatron-vlm:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 30 # Increase this timeout value as needed
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      TORCH_CUDA_ARCH_LIST: "7.5;8.0;8.9;9.0;10.0;12.0+PTX"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Reinstall FlashInfer for runtime GPU arch
+        run: |
+          unset TORCH_CUDA_ARCH_LIST
+          pip3 install --force-reinstall --no-deps flashinfer-python
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install qwen_vl_utils
+          pip3 install mathruler
+      - name: Prepare GEO3K dataset
+        run: |
+          python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k --local_save_dir ${PWD}/data/geo3k
+      - name: Running GEO3K E2E training tests with FSDP on 8 L20 GPUs (VLM)
+        run: |
+          ray stop --force
+          ROLLOUT_TP=2 \
+          INFER_BACKEND=trtllm \
+          MODEL_PATH="${HOME}/models/Qwen/Qwen3-VL-2B-Instruct" \
+            bash examples/grpo_trainer/run_qwen2_5_vl_7b_fsdp.sh \
+            trainer.total_training_steps=1 \
+            data.train_batch_size=8 \
+            data.max_response_length=64 \
+            actor_rollout_ref.actor.ppo_mini_batch_size=8 \
+            actor_rollout_ref.actor.ppo_max_token_len_per_gpu=2048 \
+            actor_rollout_ref.rollout.n=2 \
+            actor_rollout_ref.rollout.gpu_memory_utilization=0.35 \
+            actor_rollout_ref.rollout.max_model_len=2048 \
+            actor_rollout_ref.rollout.max_num_seqs=16 \
+            actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=2048 \
+            data.train_files="['${PWD}/data/geo3k/train.parquet']" \
+            data.val_files="['${PWD}/data/geo3k/test.parquet']" \
+            trainer.logger='["console"]'
+      - name: clean up
+        run: |
+          rm -rf checkpoints
+      - name: Prepare GSM8K dataset (data_preprocess)
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_save_dir ${PWD}/data/gsm8k
+      - name: Running GRPO E2E with FP8 TRT-LLM rollout (Qwen3-0.6B, gsm8k)
+        run: |
+          unset TORCH_CUDA_ARCH_LIST  # Let TRT-LLM CUTLASS DSL auto-detect runtime GPU arch
+          export TRTLLM_ENABLE_PDL=0  # Disable Programmatic Dependent Launch (uses SM 9.0 griddepcontrol)
+          ray stop --force
+          export INFER_TP=2 ACTOR_TP=2 ACTOR_PP=2 ACTOR_VPP=2 ACTOR_EP=1 ACTOR_CP=2 REF_TP=2 REF_PP=2 REF_VPP=2 REF_EP=1 REF_CP=2 GEN_MOE_TP=null GEN_MOE_EP=null
+          export NNODES=1 GPUS_PER_NODE=8 TRTLLM_MOE_BACKEND=TRITON
+          export DATA_DIR=${PWD} DAPO_MATH_TRAIN=${PWD}/data/gsm8k/train.parquet AIME_VAL=${PWD}/data/gsm8k/test.parquet MODEL_PATH=${HOME}/models/Qwen/Qwen3-0.6B
+          INFER_BACKEND=trtllm ROLLOUT_QUANTIZATION=fp8 bash examples/grpo_trainer/run_qwen3_30b_a3b_megatron.sh \
+            reward_model.reward_kwargs.overlong_buffer_cfg.len=258 \
+            reward_model.reward_kwargs.max_resp_len=512 \
+            data.max_prompt_length=128 \
+            data.max_response_length=64 \
+            data.train_batch_size=32 \
+            actor_rollout_ref.rollout.n=1 \
+            actor_rollout_ref.rollout.max_num_seqs=16 \
+            actor_rollout_ref.rollout.max_num_batched_tokens=1024 \
+            actor_rollout_ref.rollout.max_model_len=1024 \
+            actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=False \
+            actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=False \
+            trainer.total_training_steps=1 \
+            trainer.logger='["console"]'
+      - name: clean up
+        run: |
+          rm -rf checkpoints
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: [setup, trtllm_unit_tests, e2e_grpo_trainer_megatron-qwen2, e2e_grpo_trainer_megatron-vlm]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}"

verl/.github/workflows/e2e_ppo_trainer.yml

@@ -0,0 +1,78 @@
+name: e2e_ppo_trainer
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  # For push, for now only anti-patterns are specified so it is more conservative
+  # and achieves higher coverage.
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      # Other entrypoints
+      - "!verl/trainer/fsdp_sft_trainer.py"
+
+      # Megatron
+      - "!verl/workers/**/megatron_*.py"
+
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      # Other entrypoints
+      - "!**/*.md"
+      - "!docker/**"
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py"
+      # Docs
+      - "!docs/**"
+
+      # Megatron
+      - "!verl/workers/**/megatron_*.py"
+      # Entrypoints
+      - ".github/workflows/e2e_ppo_trainer.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "examples/data_preprocess/geo3k.py"
+      - "tests/special_e2e/ppo_trainer"
+      - "verl/trainer/main_ppo.py"
+      - "verl/trainer/config/ppo_trainer.yaml"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+jobs:
+  pre_commit_for_ppo:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install the current repository
+        run: |
+          pip install pre-commit hydra-core
+          pip3 install --no-deps -e .
+      - name: Set ruff --output-format=github
+        run: |
+          sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml
+          git add .pre-commit-config.yaml
+      - uses: pre-commit/action@v3.0.1
+        with:
+          extra_args: "" # Overriding default "--all-files"
+

verl/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml

@@ -0,0 +1,202 @@
+# # Tests layout
+
+# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
+# - `tests/trainer` for testing functionality related to `verl/trainer`
+# - `tests/models` for testing functionality related to `verl/models`
+# - ...
+
+# There are a few folders with `special_` prefix, created for special purposes:
+# - `special_distributed`: unit tests that must run with multiple GPUs
+# - `special_e2e`: end-to-end tests with training/generation scripts
+# - `special_npu`: tests for NPUs
+# - `special_sanity`: a suite of quick sanity tests
+# - `special_standalone`: a set of test that are designed to run in dedicated environments
+
+# Accelerators for tests
+# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
+# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.
+
+# # Workflow layout
+
+# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
+# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
+# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
+# 3. End-to-end tests: `e2e_*.yml`
+# 4. Unit tests
+#   - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py`
+#   - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
+#   - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
+#     - new workflow yaml is added to `.github/workflows`
+#     - new tests are added to workflow mentioned in 2.
+
+name: e2e_ppo_trainer_megatron_sglang
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch.
+  # For push, for now only anti-patterns are specified so it is more conservative
+  # and achieves higher coverage.
+  push:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      # Other entrypoints
+      - "!verl/trainer/fsdp_sft_trainer.py" # FSDP
+      - "!verl/workers/**/*dp_*.py"
+      - "!verl/utils/fsdp_utils.py"
+      - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py"
+      - "!verl/model_merger/fsdp_model_merger.py"
+  pull_request:
+    branches:
+      - main
+      - v0.*
+    paths:
+      - "**/*.py"
+      # Other entrypoints
+      - "!docker/**"
+      # Docs
+      - "!**/*.md"
+      - "!docs/**"
+      - "!examples/**"
+      - "!tests/**"
+      - "!verl/trainer/main_*.py"
+      - "!verl/trainer/fsdp_sft_trainer.py" # FSDP
+      - "!verl/workers/**/*dp_*.py"
+      - "!verl/utils/fsdp_utils.py"
+      - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py"
+      - "!verl/model_merger/fsdp_model_merger.py"
+      # Entrypoints
+      - "verl/worksers/rollout/sglang_rollout/*"
+      - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml"
+      - "examples/data_preprocess/gsm8k.py"
+      - "examples/data_preprocess/geo3k.py"
+      - "tests/special_e2e/run_ppo_trainer_megatron.sh"
+      - "verl/trainer/main_ppo.py"
+      - "verl/trainer/config/ppo_megatron_trainer.yaml"
+
+# Cancel jobs on the same ref if a new one is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# Declare permissions just read content.
+permissions:
+  contents: read
+
+env:
+  IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2"
+  DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
+
+jobs:
+  setup:
+    if: github.repository_owner == 'verl-project'
+    runs-on: ubuntu-latest
+    outputs:
+      runner-label: ${{ steps.create-runner.outputs.runner-label }}
+      mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: create-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "create"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-image: "${{ env.IMAGE }}"
+
+  e2e_ppo_trainer_megatron-deepseek:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 60 # Increase this timeout value as needed
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ENGINE: sglang
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install megatron-bridge --no-deps
+          pip3 install git+https://github.com/ISEEKYAN/mbridge.git@main --no-deps --no-build-isolation
+          pip3 install --no-deps -e .
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+      - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek)
+        run: |
+          ray stop --force
+          OPTIM_MEMORY_EFFICIENT=True ENGINE=sglang SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh
+      - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek)
+        run: |
+          ray stop --force
+          export VLLM_USE_V1=1
+          ray start --head
+          ENGINE=sglang MODE=async RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 bash tests/special_e2e/run_ppo_trainer_megatron.sh
+      - name: Profiling GRPO GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Deepseek)
+        run: |
+          ray stop --force
+          PROFILE_ENABLE=True ENGINE=sglang ADV_ESTIMATOR=grpo USE_DYNAMIC_BSZ=False MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh
+          if [ -z "$( ls -A '/tmp/ray/session_latest/logs/nsight/' )" ]; then
+            echo "[ERROR] not found any profiling files"
+            exit 1
+          else
+            echo "[SUCCESS] profile success"
+          fi
+      - name: clean up
+        run: |
+          rm -rf checkpoints
+
+  # Qwen3-0.6B: dense, tie_word_embeddings=True
+  e2e_ppo_trainer_megatron-qwen3:
+    needs: setup
+    runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"]
+    timeout-minutes: 60 # Increase this timeout value as needed
+    env:
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
+      HF_ENDPOINT: "https://hf-mirror.com"
+      HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
+      ENGINE: sglang
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -r requirements-test.txt
+          pip3 install --no-deps -e .
+          pip3 install megatron-bridge --no-deps
+      - name: Prepare GSM8K dataset
+        run: |
+          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
+      - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) testing learning rate scheduler
+        run: |
+          ray stop --force
+          ALL_OFFLOAD=True VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 LR_WARMUP_STEPS=1 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh
+      - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with FP8 rollout
+        run: |
+          ray stop --force
+          export VLLM_USE_V1=1
+          ROLLOUT_QUANTIZATION=fp8 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh
+      - name: clean up
+        run: |
+          rm -rf checkpoints
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs:
+      [setup, e2e_ppo_trainer_megatron-deepseek, e2e_ppo_trainer_megatron-qwen3]
+    if: always()
+    steps:
+      - id: destroy-runner
+        uses: volcengine/vemlp-github-runner@v1
+        with:
+          mode: "destroy"
+          faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
+          mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}"