Add files using upload-large-folder tool

code/RL_model/verl/verl_train/verl/experimental/agent_loop/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .agent_loop import AgentLoopBase, AgentLoopManager, AgentLoopWorker, AsyncLLMServerManager
+from .single_turn_agent_loop import SingleTurnAgentLoop
+from .tool_agent_loop import ToolAgentLoop
+
+_ = [SingleTurnAgentLoop, ToolAgentLoop]
+
+__all__ = ["AgentLoopBase", "AgentLoopManager", "AsyncLLMServerManager", "AgentLoopWorker"]

code/RL_model/verl/verl_train/verl/experimental/agent_loop/agent_loop.py

@@ -0,0 +1,1022 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import heapq
+import logging
+import os
+import random
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+from uuid import uuid4
+
+import hydra
+import numpy as np
+import ray
+import torch
+from cachetools import LRUCache
+from omegaconf import DictConfig, OmegaConf
+from PIL import Image
+from pydantic import BaseModel, ConfigDict
+from tensordict import TensorDict
+from transformers import AutoProcessor, AutoTokenizer
+
+from verl.experimental.agent_loop.prometheus_utils import update_prometheus_config
+from verl.experimental.agent_loop.utils import resolve_config_path
+from verl.experimental.reward_loop import RewardLoopWorker
+from verl.protocol import DataProto
+from verl.single_controller.ray.base import RayResourcePool, RayWorkerGroup
+from verl.utils import hf_processor, hf_tokenizer
+from verl.utils.chat_template import initialize_system_prompt
+from verl.utils.dataset.rl_dataset import RLHFDataset, get_dataset_class
+from verl.utils.fs import copy_to_local
+from verl.utils.model import compute_position_id_with_mask
+from verl.utils.ray_utils import get_event_loop
+from verl.utils.rollout_trace import (
+    RolloutTraceConfig,
+    rollout_trace_attr,
+    rollout_trace_op,
+)
+from verl.utils.transferqueue_utils import tqbridge
+from verl.workers.rollout.replica import TokenOutput, get_rollout_replica_class
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+class AsyncLLMServerManager:
+    """
+    A class to manage multiple OpenAI compatible LLM servers. This class provides
+    - Load balance: least requests load balancing
+    - Sticky session: send multi-turn chat completions to same server for automatic prefix caching
+    """
+
+    def __init__(self, config: DictConfig, server_handles: list[ray.actor.ActorHandle], max_cache_size: int = 10000):
+        """Initialize the AsyncLLMServerManager.
+
+        Args:
+            config (DictConfig): YAML config.
+            server_handles (List[ray.actor.ActorHandle]): OpenAI compatible LLM server actor handles.
+            max_cache_size (int, optional): max cache size for request_id to server mapping. Defaults to 10000.
+        """
+        self.config = config
+        self.server_handles = server_handles
+        random.shuffle(self.server_handles)
+
+        # Least requests load balancing
+        self.weighted_serveres = [[0, idx, server] for idx, server in enumerate(self.server_handles)]
+        heapq.heapify(self.weighted_serveres)
+
+        # LRU cache to map request_id to server
+        self.request_id_to_server = LRUCache(maxsize=max_cache_size)
+
+    def _choose_server(self, request_id: str) -> ray.actor.ActorHandle:
+        # TODO: implement server pressure awareness load balancing
+        if request_id in self.request_id_to_server:
+            return self.request_id_to_server[request_id]
+
+        _, _, server = self.weighted_serveres[0]
+        self.weighted_serveres[0][0] += 1
+        heapq.heapreplace(self.weighted_serveres, self.weighted_serveres[0])
+        self.request_id_to_server[request_id] = server
+        return server
+
+    @rollout_trace_op
+    async def generate(
+        self,
+        request_id,
+        *,
+        prompt_ids: list[int],
+        sampling_params: dict[str, Any],
+        image_data: Optional[list[Any]] = None,
+        video_data: Optional[list[Any]] = None,
+    ) -> TokenOutput:
+        """Generate tokens from prompt ids.
+
+        Args:
+            request_id (str): request id for sticky session.
+            prompt_ids (List[int]): List of prompt token ids.
+            sampling_params (Dict[str, Any]): Sampling parameters for the chat completion.
+
+        Returns:
+            TokenOutput: token output
+        """
+        server = self._choose_server(request_id)
+        output = await server.generate.remote(
+            request_id=uuid4().hex,  # use new request_id for each turn
+            prompt_ids=prompt_ids,
+            sampling_params=sampling_params,
+            image_data=image_data,
+            video_data=video_data,
+        )
+        return output
+
+
+class AgentLoopMetrics(BaseModel):
+    """Agent loop performance metrics."""
+
+    generate_sequences: float = 0.0
+    tool_calls: float = 0.0
+    num_preempted: int = -1  # -1 means not available
+
+
+class AgentLoopOutput(BaseModel):
+    """Agent loop output."""
+
+    prompt_ids: list[int]
+    """Prompt token ids."""
+    response_ids: list[int]
+    """Response token ids including LLM generated token, tool response token."""
+    response_mask: list[int]
+    """Response mask, 1 for LLM generated token, 0 for tool response token."""
+    response_logprobs: Optional[list[float]] = None
+    """Log probabilities for the response tokens."""
+    routed_experts: Optional[Any] = None
+    """Routed experts for the total tokens."""
+    multi_modal_data: Optional[dict[str, Any]] = None
+    """Multi-modal data for multi-modal tools."""
+    reward_score: Optional[float] = None
+    """Reward score for the trajectory."""
+    num_turns: int = 0
+    """Number of chat turns, including user, assistant, tool."""
+    metrics: AgentLoopMetrics
+    """Auxiliary performance metrics"""
+    extra_fields: dict[str, Any] = {}
+    """Extra fields for dynamic addition."""
+
+
+class _InternalAgentLoopOutput(AgentLoopOutput):
+    """Internal agent loop output with padded sequences."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    prompt_ids: torch.Tensor
+    """Padded prompt token ids."""
+    response_ids: torch.Tensor
+    """Padded response token ids."""
+    input_ids: torch.Tensor
+    """Padded input ids(prompt_ids + response_ids)."""
+    position_ids: torch.Tensor
+    """Padded position ids."""
+    response_mask: torch.Tensor
+    """Padded response mask."""
+    attention_mask: torch.Tensor
+    """Padded attention mask."""
+    response_logprobs: Optional[torch.Tensor] = None
+    """Padded log probabilities for the response tokens."""
+    routed_experts: Optional[torch.Tensor] = None
+    """Padded routed experts for the total tokens."""
+    multi_modal_inputs: Optional[dict[str, torch.Tensor]] = None
+    """Multi-modal inputs for processors (e.g., pixel_values, image_grid_thw)."""
+    extra_fields: dict[str, Any] = {}
+    """Extra fields for dynamic addition."""
+
+
+class DictConfigWrap:
+    """Wrapper for DictConfig to avoid hydra.utils.instantiate recursive resolve."""
+
+    def __init__(self, config: DictConfig):
+        self.config = config
+
+
+class AgentLoopBase(ABC):
+    """An agent loop takes an input message, chat with OpenAI compatible LLM server and interact with various
+    environments."""
+
+    def __init__(
+        self,
+        trainer_config: DictConfigWrap,
+        server_manager: AsyncLLMServerManager,
+        tokenizer: AutoTokenizer,
+        processor: AutoProcessor,
+        dataset_cls: type[RLHFDataset],
+        dataset_config: DictConfigWrap,
+        **kwargs,
+    ):
+        """Initialize agent loop, each sample will have its own loop instance.
+
+        Args:
+            trainer_config (DictConfigWrap): trainer config.
+            server_manager (AsyncLLMServerManager): OpenAI compatible LLM server manager.
+            tokenizer (AutoTokenizer): Tokenizer for tokenize messages.
+            processor (AutoProcessor): Processor for process messages.
+            dataset_cls (type[Dataset]): Dataset class for creating dataset, Defaults to RLHFDataset.
+            dataset_config (DictConfigWrap): Dataset config.
+        """
+        self.config = trainer_config.config
+        self.server_manager = server_manager
+        self.tokenizer = tokenizer
+        self.processor = processor
+        self.dataset_cls = dataset_cls
+        self.dataset_config = dataset_config.config
+        self.apply_chat_template_kwargs = self.dataset_config.get("apply_chat_template_kwargs", {})
+        self.system_prompt = initialize_system_prompt(self.tokenizer, **self.apply_chat_template_kwargs)
+        self.loop = get_event_loop()
+
+    async def process_vision_info(self, messages: list[dict]) -> dict:
+        """Extract images and videos from messages.
+
+        Args:
+            messages (list[dict]): Input messages.
+
+        Returns:
+            dict: Multi-modal data with keys "images" and "videos".
+        """
+        multi_modal_data = {}
+        if self.processor is not None:
+            images, videos = await self.dataset_cls.process_vision_info(
+                messages, image_patch_size=self.processor.image_processor.patch_size, config=self.dataset_config
+            )
+            if images is not None:
+                multi_modal_data["images"] = images
+            if videos is not None:
+                multi_modal_data["videos"] = videos
+
+        return multi_modal_data
+
+    async def apply_chat_template(
+        self,
+        messages: list[dict],
+        tools: list[dict] = None,
+        images: list[Image.Image] = None,
+        videos: list[tuple[torch.Tensor, dict]] = None,
+        remove_system_prompt: bool = False,
+    ):
+        """Apply chat template to messages with optional tools, images, and videos.
+
+        Args:
+            messages (list[dict]): Input messages.
+            tools (list[dict], optional): Tools schemas. Defaults to None.
+            images (list[Image.Image], optional): Input images. Defaults to None.
+            videos (list[tuple[torch.Tensor, dict]], optional): Input videos. Defaults to None.
+            remove_system_prompt (bool, optional): Whether to remove system prompt. Defaults to False.
+
+        Returns:
+            list[int]: Prompt token ids.
+        """
+        if self.processor is not None:
+            raw_prompt = await self.loop.run_in_executor(
+                None,
+                lambda: self.processor.apply_chat_template(
+                    messages,
+                    tools=tools,
+                    add_generation_prompt=True,
+                    tokenize=False,
+                    **self.apply_chat_template_kwargs,
+                ),
+            )
+
+            # split the videos and according metadatas
+            if videos is not None:
+                videos, video_metadatas = zip(*videos, strict=False)
+                videos, video_metadatas = list(videos), list(video_metadatas)
+            else:
+                video_metadatas = None
+
+            model_inputs = self.processor(
+                text=[raw_prompt],
+                images=images,
+                videos=videos,
+                video_metadatas=video_metadatas,
+                return_tensors="pt",
+                do_sample_frames=False,
+            )
+            prompt_ids = model_inputs.pop("input_ids").squeeze(0).tolist()
+        else:
+            prompt_ids = await self.loop.run_in_executor(
+                None,
+                lambda: self.tokenizer.apply_chat_template(
+                    messages,
+                    tools=tools,
+                    add_generation_prompt=True,
+                    tokenize=True,
+                    **self.apply_chat_template_kwargs,
+                ),
+            )
+
+        if remove_system_prompt:
+            prompt_ids = prompt_ids[len(self.system_prompt) :]
+
+        return prompt_ids
+
+    @abstractmethod
+    async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput:
+        """Run agent loop to interact with LLM server and environment.
+
+        Args:
+            sampling_params (Dict[str, Any]): LLM sampling params.
+            **kwargs: dataset fields from `verl.utils.dataset.RLHFDataset`.
+
+        Returns:
+            AgentLoopOutput: Agent loop output.
+        """
+        raise NotImplementedError
+
+
+"""Agent loop registry: key is agent_name, value is a dict of agent loop config
+used by hydra.utils.instantiate to initialize agent loop instance.
+
+https://hydra.cc/docs/advanced/instantiate_objects/overview/
+"""
+_agent_loop_registry: dict[str, dict] = {}
+
+
+def register(agent_name: str):
+    """Register agent loop class."""
+
+    def decorator(subclass: type[AgentLoopBase]) -> type[AgentLoopBase]:
+        fqdn = f"{subclass.__module__}.{subclass.__qualname__}"
+        _agent_loop_registry[agent_name] = {"_target_": fqdn}
+        return subclass
+
+    return decorator
+
+
+class AgentLoopWorker:
+    """Agent loop worker takes a batch of messages and run each message in an agent loop."""
+
+    def __init__(
+        self,
+        config: DictConfig,
+        server_handles: list[ray.actor.ActorHandle],
+        reward_router_address: str = None,
+    ):
+        """Initialize agent loop manager.
+        Args:
+            config (DictConfig): YAML config.
+            server_handles (List[ray.actor.ActorHandle]): OpenAI compatible LLM server actor handles.
+            reward_router_address (str): reward router address.
+        """
+        self.config = config
+
+        # for recipe to change
+        if not hasattr(self, "server_manager"):
+            self.server_manager = AsyncLLMServerManager(config, server_handles)
+
+        self.dataset_cls = get_dataset_class(config.data)
+        self.reward_router_address = reward_router_address
+
+        model_path = config.actor_rollout_ref.model.path
+        self.model_name = "/".join(model_path.split("/")[-2:])
+        local_path = copy_to_local(config.actor_rollout_ref.model.path)
+        self.tokenizer = hf_tokenizer(local_path, trust_remote_code=True)
+        self.processor = hf_processor(local_path, trust_remote_code=True)
+
+        agent_loop_config_path = config.actor_rollout_ref.rollout.agent.agent_loop_config_path
+        if agent_loop_config_path:
+            resolved_path = resolve_config_path(agent_loop_config_path)
+            agent_loop_configs = OmegaConf.load(resolved_path)
+            for agent_loop_config in agent_loop_configs:
+                _agent_loop_registry[agent_loop_config.name] = agent_loop_config
+        if self.config.actor_rollout_ref.model.get("custom_chat_template", None) is not None:
+            if self.processor is not None:
+                self.processor.chat_template = self.config.actor_rollout_ref.model.custom_chat_template
+            self.tokenizer.chat_template = self.config.actor_rollout_ref.model.custom_chat_template
+
+        use_reward_loop = True if self.config.reward_model.use_reward_loop else None
+        self.use_reward_loop = use_reward_loop
+        if use_reward_loop and not hasattr(self, "reward_loop_worker"):
+            self.reward_loop_worker = RewardLoopWorker.options(
+                scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
+                    node_id=ray.get_runtime_context().get_node_id(),
+                    soft=False,
+                ),
+            ).remote(self.config, self.reward_router_address)
+
+        trace_config = self.config.actor_rollout_ref.rollout.get("trace", {})
+        RolloutTraceConfig.init(
+            self.config.trainer.project_name,
+            self.config.trainer.experiment_name,
+            trace_config.get("backend"),
+            trace_config.get("token2text", False),
+            trace_config.get("max_samples_per_step_per_worker", None),
+        )
+
+    @tqbridge()
+    async def generate_sequences(self, batch: DataProto) -> DataProto:
+        """Generate sequences from agent loop.
+
+        Args:
+            batch (DataProto): Input batch.
+
+        Returns:
+            DataProto: Output batch.
+            - prompts: [bsz, prompt_length], prompt token ids from dataset.
+            - responses: [bsz, response_length], output token ids include response tokens
+              from LLM generation and observation tokens from tool_calls.
+            - response_mask: [bsz, response_length], 1 for LLM generated tokens, 0 for observation/padding tokens.
+            - input_ids: [bsz, prompt_length + response_length], whole sequence token ids, including prompt tokens
+              and response tokens.
+            - attention_mask: [bsz, prompt_length + response_length], 0 for padding tokens, 1 for other tokens.
+            - position_ids: [bsz, prompt_length + response_length], incremental position ids.
+
+            For multi-turn conversations:
+            responses:     |<- LLM generation ->|<- tool_calls ->|<- LLM generation ->|<- padding ->|
+            response_mask: | 1, 1, 1, ..., 1, 1 | 0, 0, .., 0, 0 | 1, 1, 1, ..., 1, 1 | 0, 0, ..., 0|
+        """
+        config = self.config.actor_rollout_ref.rollout
+        sampling_params = dict(
+            temperature=config.temperature,
+            top_p=config.top_p,
+            top_k=config.top_k,
+            repetition_penalty=1.0,
+            logprobs=config.calculate_log_probs,
+        )
+
+        # override sampling params for validation
+        if batch.meta_info.get("validate", False):
+            sampling_params["top_p"] = config.val_kwargs.top_p
+            sampling_params["top_k"] = config.val_kwargs.top_k
+            sampling_params["temperature"] = config.val_kwargs.temperature
+
+        # by default, we assume it's a single turn agent
+        if "agent_name" not in batch.non_tensor_batch:
+            default_agent_loop = config.agent.default_agent_loop
+            batch.non_tensor_batch["agent_name"] = np.array([default_agent_loop] * len(batch), dtype=object)
+
+        if "index" in batch.non_tensor_batch:
+            index = batch.non_tensor_batch["index"]
+        else:
+            index = np.arange(len(batch))
+
+        max_samples_per_worker = RolloutTraceConfig.get_instance().max_samples_per_step_per_worker
+
+        # For n rollouts per sample, we trace all n rollouts for selected samples
+        # Note: This sampling happens per-worker, so total traces = max_samples_per_worker * num_workers * n
+        if max_samples_per_worker is not None:
+            unique_sample_indices = np.unique(index)
+            if max_samples_per_worker < len(unique_sample_indices):
+                selected_samples = set(
+                    np.random.choice(unique_sample_indices, max_samples_per_worker, replace=False).tolist()
+                )
+                traced_indices = set(i for i in range(len(batch)) if index[i] in selected_samples)
+            else:
+                traced_indices = set(range(len(batch)))
+        else:
+            traced_indices = set(range(len(batch)))
+
+        trajectory_info = await get_trajectory_info(
+            batch.meta_info.get("global_steps", -1), index.tolist(), batch.meta_info.get("validate", False)
+        )
+
+        tasks = []
+        for i in range(len(batch)):
+            trace_this_sample = i in traced_indices
+            kwargs = {k: v[i] for k, v in batch.non_tensor_batch.items()}
+            tasks.append(
+                asyncio.create_task(
+                    self._run_agent_loop(sampling_params, trajectory_info[i], trace=trace_this_sample, **kwargs)
+                )
+            )
+        outputs = await asyncio.gather(*tasks)
+
+        output = self._postprocess(outputs)
+
+        return output
+
+    async def _run_agent_loop(
+        self,
+        sampling_params: dict[str, Any],
+        trajectory: dict[str, Any],
+        *,
+        agent_name: str,
+        trace: bool = True,
+        **kwargs,
+    ) -> _InternalAgentLoopOutput:
+        with rollout_trace_attr(
+            step=trajectory["step"],
+            sample_index=trajectory["sample_index"],
+            rollout_n=trajectory["rollout_n"],
+            validate=trajectory["validate"],
+            name="agent_loop",
+            trace=trace,
+        ):
+            assert agent_name in _agent_loop_registry, (
+                f"Agent loop {agent_name} not registered, registered agent loops: {_agent_loop_registry.keys()}"
+            )
+
+            agent_loop_config = _agent_loop_registry[agent_name]
+            agent_loop = hydra.utils.instantiate(
+                config=agent_loop_config,
+                trainer_config=DictConfigWrap(config=self.config),
+                server_manager=self.server_manager,
+                tokenizer=self.tokenizer,
+                processor=self.processor,
+                dataset_cls=self.dataset_cls,
+                dataset_config=DictConfigWrap(self.config.data),
+            )
+            output: AgentLoopOutput = await agent_loop.run(sampling_params, **kwargs)
+            return await self._agent_loop_postprocess(output, **kwargs)
+
+    async def _agent_loop_postprocess(self, output, **kwargs) -> _InternalAgentLoopOutput:
+        """Perform post-processing operations on the output of each individual agent loop."""
+        output.extra_fields["raw_prompt"] = kwargs["raw_prompt"]
+
+        # Some AgentLoop may have already computed the reward score, e.g SWE-agent.
+
+        # NOTE: consistent with the legacy batch version of generate_sequences that existed in the
+        # deprecated vLLM SPMD rollout implementation.
+        # prompt_ids: left padded with zeros (e.g., [0,0,0,0,1,2,3,4])
+        # response_ids: right padded with zeros (e.g., [5,6,7,8,0,0,0,0])
+        # input_ids: concatenation of prompt + response
+        # Mask:
+        # For example, if the prompt is [1,2,3,4] and the response is [5,6,7,(tool start)8,9(tool end),10,11,12]
+        # - prompt_attention_mask: 0s for padding, 1s for tokens
+        #   e.g., [0,0,0,0,1,1,1,1]
+        # - response_attention_mask: 0s for padding, 1s for tokens
+        #   e.g., [1,1,1,1,1,1,1,1,1,1,1,0,0,0,0]
+        # attention_mask: concatenation of prompt_attention_mask and response_attention_mask
+        #   e.g., [0,0,0,0,1,1,1,1(prompt),1,1,1,1,1,1,1,1,1,1,1,0,0,0,0(response)]
+        # - response_mask: 1s for LLM generated tokens, 0 for tool response/padding tokens
+        #   e.g., [1,1,1,1,1,1,1,(tool start),0,0(tool end),1,1,0,0,0,0]
+        # - position_ids: sequential positions for tokens, starting at 0
+        #   e.g., [0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0,0,0,0]
+
+        # TODO(wuxibin): remove padding and use tensordict.
+        self.tokenizer.padding_side = "left"
+        prompt_output = self.tokenizer.pad(
+            {"input_ids": output.prompt_ids},
+            padding="max_length",
+            max_length=self.config.actor_rollout_ref.rollout.prompt_length,
+            return_tensors="pt",
+            return_attention_mask=True,
+        )
+        if prompt_output["input_ids"].dim() == 1:
+            prompt_output["input_ids"] = prompt_output["input_ids"].unsqueeze(0)
+            prompt_output["attention_mask"] = prompt_output["attention_mask"].unsqueeze(0)
+
+        self.tokenizer.padding_side = "right"
+        response_output = self.tokenizer.pad(
+            {"input_ids": output.response_ids},
+            padding="max_length",
+            max_length=self.config.actor_rollout_ref.rollout.response_length,
+            return_tensors="pt",
+            return_attention_mask=True,
+        )
+        if response_output["input_ids"].dim() == 1:
+            response_output["input_ids"] = response_output["input_ids"].unsqueeze(0)
+            response_output["attention_mask"] = response_output["attention_mask"].unsqueeze(0)
+
+        response_mask_output = self.tokenizer.pad(
+            {"input_ids": output.response_mask},
+            padding="max_length",
+            max_length=self.config.actor_rollout_ref.rollout.response_length,
+            return_tensors="pt",
+            return_attention_mask=False,
+        )
+        if response_mask_output["input_ids"].dim() == 1:
+            response_mask_output["input_ids"] = response_mask_output["input_ids"].unsqueeze(0)
+
+        response_logprobs = None
+        if output.response_logprobs is not None:
+            pad_size = self.config.actor_rollout_ref.rollout.response_length - len(output.response_logprobs)
+            response_logprobs = torch.tensor(output.response_logprobs + [0.0] * pad_size).unsqueeze(0)
+
+        response_mask = response_mask_output["input_ids"] * response_output["attention_mask"]
+        attention_mask = torch.cat([prompt_output["attention_mask"], response_output["attention_mask"]], dim=1)
+        input_ids = torch.cat([prompt_output["input_ids"], response_output["input_ids"]], dim=1)
+
+        routed_experts = None
+        if output.routed_experts is not None:
+            total_length = input_ids.shape[1]
+            length, layer_num, topk_num = output.routed_experts.shape
+            if isinstance(output.routed_experts, np.ndarray):
+                experts_tensor = torch.from_numpy(output.routed_experts)
+            elif isinstance(output.routed_experts, torch.Tensor):
+                experts_tensor = output.routed_experts
+            else:
+                raise TypeError(f"Unsupported type for routed_experts: {type(output.routed_experts)}")
+            routed_experts = torch.zeros(1, total_length, layer_num, topk_num, dtype=experts_tensor.dtype)
+
+            # Calculate start position: left padding means original prompt starts at the end
+            start_pos = prompt_output["input_ids"].shape[1] - len(output.prompt_ids)
+            end_pos = min(start_pos + length, total_length)
+
+            # Add boundary checks for robustness
+            if start_pos < 0 or end_pos > total_length:
+                raise ValueError(
+                    f"Invalid position range: start_pos={start_pos}, end_pos={end_pos}, total_length={total_length}"
+                )
+
+            routed_experts[:, start_pos:end_pos] = experts_tensor.unsqueeze(0)
+
+        multi_modal_inputs = self._compute_multi_modal_inputs(output, input_ids)
+        position_ids = self._compute_position_ids(input_ids, attention_mask, multi_modal_inputs)
+        await self._compute_score(
+            output,
+            prompts=prompt_output["input_ids"],
+            responses=response_output["input_ids"],
+            attention_mask=attention_mask,
+            input_ids=input_ids,
+            position_ids=position_ids,
+            kwargs=kwargs,
+        )
+
+        return _InternalAgentLoopOutput(
+            prompt_ids=prompt_output["input_ids"],
+            response_ids=response_output["input_ids"],
+            input_ids=input_ids,
+            position_ids=position_ids,
+            response_mask=response_mask,
+            attention_mask=attention_mask,
+            response_logprobs=response_logprobs,
+            routed_experts=routed_experts,
+            multi_modal_inputs=multi_modal_inputs,
+            multi_modal_data=output.multi_modal_data,
+            reward_score=output.reward_score,
+            num_turns=output.num_turns,
+            metrics=output.metrics,
+            extra_fields=output.extra_fields,
+        )
+
+    def _compute_multi_modal_inputs(self, output, input_ids) -> dict[str, torch.Tensor]:
+        """Compute multi-modal inputs with image and video."""
+        multi_modal_inputs = {}
+        if self.processor is None:
+            return multi_modal_inputs
+
+        images = output.multi_modal_data.get("images")
+        videos = output.multi_modal_data.get("videos")
+        # split the videos and according metadatas
+        if videos is not None:
+            videos, video_metadatas = zip(*videos, strict=False)
+            videos, video_metadatas = list(videos), list(video_metadatas)
+        else:
+            video_metadatas = None
+        current_text = self.tokenizer.decode(input_ids.squeeze(0), skip_special_tokens=True)
+        multi_modal_inputs = self.processor(
+            text=[current_text],
+            images=images,
+            videos=videos,
+            video_metadatas=video_metadatas,
+            return_tensors="pt",
+            do_sample_frames=False,
+        )
+        multi_modal_inputs.pop("input_ids", None)
+        multi_modal_inputs.pop("attention_mask", None)
+
+        # We must use dict(multi_modal_inputs) to convert BatchFeature values to a new dict
+        # because np.array() only keeps the keys for BatchFeature.
+        multi_modal_inputs = dict(multi_modal_inputs.convert_to_tensors("pt"))
+        image_grid_thw = multi_modal_inputs.get("image_grid_thw")
+        if image_grid_thw is not None:
+            images_seqlens = torch.repeat_interleave(image_grid_thw[:, 1] * image_grid_thw[:, 2], image_grid_thw[:, 0])
+            multi_modal_inputs["images_seqlens"] = images_seqlens
+        return multi_modal_inputs
+
+    def _compute_position_ids(self, input_ids, attention_mask, multi_modal_inputs) -> torch.Tensor:
+        """Compute position ids for multi-modal inputs."""
+        if self.processor is None:
+            return compute_position_id_with_mask(attention_mask)  # (1, seq_len)
+
+        image_grid_thw = multi_modal_inputs.get("image_grid_thw")
+        video_grid_thw = multi_modal_inputs.get("video_grid_thw")
+
+        # Model's get_rope_index has been dynamically bind to the processor.
+        vision_position_ids, _ = self.processor.get_rope_index(
+            input_ids=input_ids,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            attention_mask=attention_mask,
+        )
+        vision_position_ids = vision_position_ids.transpose(0, 1)  # (3, 1, seq_len) => (1, 3, seq_len)
+
+        valid_mask = attention_mask[0].bool()
+        text_position_ids = torch.ones((1, len(input_ids[0])), dtype=torch.long)
+        text_position_ids[0, valid_mask] = torch.arange(valid_mask.sum().item())
+        text_position_ids = text_position_ids.unsqueeze(0)
+        position_ids = torch.cat((text_position_ids, vision_position_ids), dim=1)  # (1, 4, seq_length)
+        return position_ids
+
+    async def _compute_score(self, output, prompts, responses, attention_mask, input_ids, position_ids, kwargs):
+        """Compute reward score for single sample."""
+        enable_async_reward = (
+            self.reward_router_address is not None and self.config.reward_model.enable_resource_pool
+        ) or not self.config.reward_model.enable
+
+        if output.reward_score is None and enable_async_reward and self.use_reward_loop:
+            batch = TensorDict(
+                {
+                    "prompts": prompts,  # [1, prompt_length]
+                    "responses": responses,  # [1, response_length]
+                    "attention_mask": attention_mask,  # [1, prompt_length + response_length]
+                    "input_ids": input_ids,  # [1, prompt_length + response_length]
+                    "position_ids": position_ids,
+                },
+                batch_size=1,
+            )
+            non_tensor_batch = {
+                **{k: np.array([v]) for k, v in kwargs.items()},
+                "__num_turns__": np.array([output.num_turns]),
+                "tool_extra_fields": np.array([output.extra_fields], dtype=object),
+            }
+
+            data = DataProto(
+                batch=batch,
+                non_tensor_batch=non_tensor_batch,
+            )
+            result = await self.reward_loop_worker.compute_score.remote(data)
+            output.reward_score = result["reward_score"]
+            output.extra_fields["reward_extra_info"] = result["reward_extra_info"]
+
+    def _postprocess(self, inputs: list[_InternalAgentLoopOutput]) -> DataProto:
+        """Process the padded outputs from _run_agent_loop and combine them into a batch."""
+        # Convert lists back to tensors and stack them to create a batch.
+        prompt_ids = torch.cat([input.prompt_ids for input in inputs], dim=0)
+        response_ids = torch.cat([input.response_ids for input in inputs], dim=0)
+        response_mask = torch.cat([input.response_mask for input in inputs], dim=0)
+        attention_mask = torch.cat([input.attention_mask for input in inputs], dim=0)
+        input_ids = torch.cat([input.input_ids for input in inputs], dim=0)
+        position_ids = torch.cat([input.position_ids for input in inputs], dim=0)
+        optional_outputs = {}
+        if inputs[0].response_logprobs is not None:
+            optional_outputs["rollout_log_probs"] = torch.cat([input.response_logprobs for input in inputs], dim=0)
+        if inputs[0].routed_experts is not None:
+            optional_outputs["routed_experts"] = torch.cat([input.routed_experts for input in inputs], dim=0)
+
+        batch = TensorDict(
+            {
+                "prompts": prompt_ids,  # [bsz, prompt_length]
+                "responses": response_ids,  # [bsz, response_length]
+                "response_mask": response_mask,  # [bsz, response_length]
+                "input_ids": input_ids,  # [bsz, prompt_length + response_length]
+                "attention_mask": attention_mask,  # [bsz, prompt_length + response_length]
+                # position_ids: [bsz, 3, prompt_length + response_length] or [bsz, prompt_length + response_length]
+                "position_ids": position_ids,
+                **optional_outputs,
+            },
+            batch_size=len(inputs),
+        )
+
+        scores = [input.reward_score for input in inputs]
+        if all(score is not None for score in scores):
+            prompt_length = prompt_ids.size(1)
+            response_length = attention_mask[:, prompt_length:].sum(dim=1) - 1
+            rm_scores = torch.zeros_like(response_mask, dtype=torch.float32)
+            rm_scores[torch.arange(response_mask.size(0)), response_length] = torch.tensor(scores, dtype=torch.float32)
+            batch["rm_scores"] = rm_scores
+
+        non_tensor_batch = {
+            "__num_turns__": np.array([input.num_turns for input in inputs], dtype=np.int32),
+        }
+
+        # add reward_extra_info to non_tensor_batch
+        reward_extra_infos = [input.extra_fields.get("reward_extra_info", {}) for input in inputs]
+        reward_extra_keys = list(reward_extra_infos[0].keys())
+        for key in reward_extra_keys:
+            non_tensor_batch[key] = np.array([info[key] for info in reward_extra_infos])
+
+        # Add multi_modal_inputs to non_tensor_batch if any samples have them
+        multi_modal_inputs_list = [input.multi_modal_inputs for input in inputs]
+        if any(mmi is not None for mmi in multi_modal_inputs_list):
+            non_tensor_batch["multi_modal_inputs"] = np.array(multi_modal_inputs_list, dtype=object)
+
+        metrics = [input.metrics.model_dump() for input in inputs]
+        # Collect extra fields from all inputs and convert them to np.ndarray
+        extra_fields = {}
+        all_keys = set(key for input_item in inputs for key in input_item.extra_fields)
+        for key in all_keys:
+            temp_arr = np.empty(len(inputs), dtype=object)
+            temp_arr[:] = [input.extra_fields.get(key) for input in inputs]
+            extra_fields[key] = temp_arr
+
+        non_tensor_batch.update(extra_fields)
+
+        # Only include reward_extra_keys in meta_info if rm_scores is in batch
+        # This avoids conflicts when reward_tensor is merged later in ray_trainer.py
+        if "rm_scores" in batch.keys():
+            meta_info = {"metrics": metrics, "reward_extra_keys": reward_extra_keys}
+        else:
+            meta_info = {"metrics": metrics}
+
+        return DataProto(
+            batch=batch,
+            non_tensor_batch=non_tensor_batch,
+            meta_info=meta_info,
+        )
+
+    def create_transferqueue_client(
+        self,
+    ):
+        """Create a client for data system (TransferQueue)."""
+        from verl.single_controller.ray.base import get_random_string
+        from verl.utils.transferqueue_utils import create_transferqueue_client
+
+        client_name = get_random_string(length=6)
+
+        self.tq_client = create_transferqueue_client(
+            client_id=f"AgentLoopWorker_{client_name}",
+            config=self.config.transfer_queue,
+        )
+
+
+async def get_trajectory_info(step, index, validate):
+    """Get trajectory info.
+
+    Args:
+        step (int): global steps in the trainer.
+        index (list): form datastore extra_info.index column.
+        validate (bool): whether is a validate step.
+
+    Returns:
+        list: trajectory.
+    """
+    trajectory_info = []
+    rollout_n = 0
+    for i in range(len(index)):
+        if i > 0 and index[i - 1] == index[i]:
+            rollout_n += 1
+        else:
+            rollout_n = 0
+        trajectory_info.append({"step": step, "sample_index": index[i], "rollout_n": rollout_n, "validate": validate})
+    return trajectory_info
+
+
+class AgentLoopManager:
+    """Agent loop manager that manages a group of agent loop workers."""
+
+    def __init__(
+        self,
+        config: DictConfig,
+        worker_group: RayWorkerGroup = None,
+        rollout_resource_pool: RayResourcePool = None,
+        rm_resource_pool: RayResourcePool = None,
+    ):
+        """Initialize agent loop manager.
+
+        Args:
+            config (DictConfig): trainer config.
+            worker_group (RayWorkerGroup): ActorRolloutRef worker group for hybrid mode; None for standalone mode.
+            rollout_resource_pool (RayResourcePool): Resource pool for actor rollout (Colocate or Standalone mode).
+            rm_resource_pool (RayResourcePool): Resource pool for reward model (Standalone mode).
+        """
+        self.config = config
+        self.worker_group = worker_group
+        self.reward_model_manager = None
+        self.reward_router_address = None
+        if self.config.reward_model.enable and self.config.reward_model.enable_resource_pool:
+            from verl.experimental.reward_loop import RewardModelManager
+
+            self.reward_model_manager = RewardModelManager(config.reward_model, rm_resource_pool)
+            self.reward_router_address = self.reward_model_manager.get_router_address()
+
+        # for recipe to change
+        if not hasattr(self, "rollout_replica_class"):
+            self.rollout_replica_class = get_rollout_replica_class(self.config.actor_rollout_ref.rollout.name)
+        if not hasattr(self, "agent_loop_workers_class"):
+            self.agent_loop_workers_class = ray.remote(AgentLoopWorker)
+
+        self._initialize_llm_servers(rollout_resource_pool)
+        self._init_agent_loop_workers()
+
+    def _initialize_llm_servers(self, rollout_resource_pool: RayResourcePool):
+        rollout_world_size = (
+            self.config.actor_rollout_ref.rollout.tensor_model_parallel_size
+            * self.config.actor_rollout_ref.rollout.data_parallel_size
+            * self.config.actor_rollout_ref.rollout.pipeline_model_parallel_size
+        )
+        world_size = (
+            self.worker_group.world_size
+            if self.worker_group
+            else self.config.trainer.n_gpus_per_node * self.config.trainer.nnodes
+        )
+        num_replicas = world_size // rollout_world_size
+
+        rollout_config = self.config.actor_rollout_ref.rollout
+        model_config = self.config.actor_rollout_ref.model
+        self.rollout_replicas = [
+            self.rollout_replica_class(
+                replica_rank=replica_rank,
+                config=rollout_config,
+                model_config=model_config,
+                gpus_per_node=self.config.trainer.n_gpus_per_node,
+            )
+            for replica_rank in range(num_replicas)
+        ]
+
+        if self.worker_group and rollout_config.name != "trtllm":
+            self._run_all([server.init_hybrid(self.worker_group) for server in self.rollout_replicas])
+        elif self.worker_group and rollout_config.name == "trtllm":
+            self._run_all(
+                [
+                    server.init_hybrid_colocated(self.worker_group, rollout_resource_pool)
+                    for server in self.rollout_replicas
+                ]
+            )
+        else:
+            self._run_all([server.init_standalone() for server in self.rollout_replicas])
+
+        self.server_handles = [server._server_handle for server in self.rollout_replicas]
+        self.server_addresses = [server._server_address for server in self.rollout_replicas]
+
+        print(f"AgentLoopManager: {self.server_addresses}")
+
+        # Update Prometheus configuration with server addresses
+        if rollout_config.prometheus.enable:
+            if rollout_config.disable_log_stats:
+                raise ValueError("PROMETHEUS needs disable_log_stats==False, but it is currently True.")
+            update_prometheus_config(rollout_config.prometheus, self.server_addresses, rollout_config.name)
+
+    def _init_agent_loop_workers(self):
+        self.agent_loop_workers = []
+        num_workers = self.config.actor_rollout_ref.rollout.agent.num_workers
+
+        node_ids = [node["NodeID"] for node in ray.nodes() if node["Alive"] and node["Resources"].get("CPU", 0) > 0]
+        for i in range(num_workers):
+            # Round-robin scheduling over the all nodes
+            node_id = node_ids[i % len(node_ids)]
+            self.agent_loop_workers.append(
+                self.agent_loop_workers_class.options(
+                    name=f"agent_loop_worker_{i}" + f"_{uuid4().hex[:8]}",
+                    scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
+                        node_id=node_id, soft=True
+                    ),
+                ).remote(self.config, self.server_handles, self.reward_router_address)
+            )
+
+    def generate_sequences(self, prompts: DataProto) -> DataProto:
+        """Split input batch and dispatch to agent loop workers.
+
+        Args:
+            prompts (DataProto): Input batch.
+
+        Returns:
+            DataProto: Output batch.
+        """
+
+        # TODO: move reward_model_manager out of agent_loop manager
+        if self.reward_model_manager:
+            self.reward_model_manager.wake_up()
+
+        chunkes = prompts.chunk(len(self.agent_loop_workers))
+        outputs = ray.get(
+            [
+                worker.generate_sequences.remote(chunk)
+                for worker, chunk in zip(self.agent_loop_workers, chunkes, strict=True)
+            ]
+        )
+        output = DataProto.concat(outputs)
+        if self.reward_model_manager:
+            self.reward_model_manager.sleep()
+
+        # calculate performance metrics
+        metrics = [output.meta_info.pop("metrics") for output in outputs]  # List[List[Dict[str, str]]]
+        timing = self._performance_metrics(metrics, output)
+
+        output.meta_info = {"timing": timing, **outputs[0].meta_info}
+        return output
+
+    def _performance_metrics(self, metrics: list[list[dict[str, str]]], output: DataProto) -> dict[str, float]:
+        timing = {}
+        t_generate_sequences = np.array([metric["generate_sequences"] for chunk in metrics for metric in chunk])
+        t_tool_calls = np.array([metric["tool_calls"] for chunk in metrics for metric in chunk])
+        num_preempted = np.array([metric["num_preempted"] for chunk in metrics for metric in chunk])
+        timing["agent_loop/num_preempted/min"] = num_preempted.min()
+        timing["agent_loop/num_preempted/max"] = num_preempted.max()
+        timing["agent_loop/num_preempted/mean"] = num_preempted.mean()
+        timing["agent_loop/generate_sequences/min"] = t_generate_sequences.min()
+        timing["agent_loop/generate_sequences/max"] = t_generate_sequences.max()
+        timing["agent_loop/generate_sequences/mean"] = t_generate_sequences.mean()
+        timing["agent_loop/tool_calls/min"] = t_tool_calls.min()
+        timing["agent_loop/tool_calls/max"] = t_tool_calls.max()
+        timing["agent_loop/tool_calls/mean"] = t_tool_calls.mean()
+
+        # batch sequence generation is bounded by the slowest sample
+        slowest = np.argmax(t_generate_sequences + t_tool_calls)
+        attention_mask = output.batch["attention_mask"][slowest]
+        prompt_length = output.batch["prompts"].shape[1]
+        timing["agent_loop/slowest/generate_sequences"] = t_generate_sequences[slowest]
+        timing["agent_loop/slowest/tool_calls"] = t_tool_calls[slowest]
+        timing["agent_loop/slowest/prompt_length"] = attention_mask[:prompt_length].sum().item()
+        timing["agent_loop/slowest/response_length"] = attention_mask[prompt_length:].sum().item()
+        timing["agent_loop/slowest/num_preempted"] = num_preempted[slowest]
+
+        return timing
+
+    def clear_kv_cache(self):
+        """Clear all rollout kv cache, but don`t sleep."""
+        self._run_all([replica.clear_kv_cache() for replica in self.rollout_replicas])
+
+    def start_profile(self, **kwargs):
+        """Start profiling on all rollout replicas."""
+        self._run_all([replica.start_profile(**kwargs) for replica in self.rollout_replicas])
+
+    def stop_profile(self):
+        """Stop profiling on all rollout replicas."""
+        self._run_all([replica.stop_profile() for replica in self.rollout_replicas])
+
+    def _run_all(self, tasks: list[asyncio.Task]):
+        async def run_all():
+            await asyncio.gather(*tasks)
+
+        asyncio.run(run_all())

code/RL_model/verl/verl_train/verl/experimental/agent_loop/prometheus_utils.py

@@ -0,0 +1,110 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import os
+
+import ray
+import yaml
+
+from verl.workers.config.rollout import PrometheusConfig
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+def update_prometheus_config(config: PrometheusConfig, server_addresses: list[str], rollout_name: str | None = None):
+    """
+    Update Prometheus configuration file with server addresses and reload on first node.
+
+    server_addresses: vllm or sglang server addresses
+
+    rollout_name: name of the rollout backend (e.g., "vllm", "sglang")
+    """
+
+    if not server_addresses:
+        logger.warning("No server addresses available to update Prometheus config")
+        return
+
+    try:
+        # Get Prometheus config file path from environment or use default
+        prometheus_config_json = {
+            "global": {"scrape_interval": "10s", "evaluation_interval": "10s"},
+            "scrape_configs": [
+                {
+                    "job_name": "ray",
+                    "file_sd_configs": [{"files": ["/tmp/ray/prom_metrics_service_discovery.json"]}],
+                },
+                {"job_name": "rollout", "static_configs": [{"targets": server_addresses}]},
+            ],
+        }
+
+        # Write configuration file to all nodes
+        @ray.remote(num_cpus=0)
+        def write_config_file(config_data, config_path):
+            os.makedirs(os.path.dirname(config_path), exist_ok=True)
+            with open(config_path, "w") as f:
+                yaml.dump(config_data, f, default_flow_style=False, indent=2)
+            return True
+
+        # Reload Prometheus on all nodes. Only master node should succeed, skip errors on other nodes.
+        @ray.remote(num_cpus=0)
+        def reload_prometheus(port):
+            import socket
+            import subprocess
+
+            hostname = socket.gethostname()
+            ip_address = socket.gethostbyname(hostname)
+
+            reload_url = f"http://{ip_address}:{port}/-/reload"
+
+            try:
+                subprocess.run(["curl", "-X", "POST", reload_url], capture_output=True, text=True, timeout=10)
+                print(f"Reloading Prometheus on node: {reload_url}")
+            except Exception:
+                # Skip errors on non-master nodes
+                pass
+
+        # Get all available nodes and schedule tasks on each node
+        nodes = ray.nodes()
+        alive_nodes = [node for node in nodes if node["Alive"]]
+
+        # Write config files on all nodes
+        write_tasks = []
+        for node in alive_nodes:
+            node_ip = node["NodeManagerAddress"]
+            task = write_config_file.options(
+                resources={"node:" + node_ip: 0.001}  # Schedule to specific node
+            ).remote(prometheus_config_json, config.file)
+            write_tasks.append(task)
+
+        ray.get(write_tasks)
+
+        server_type = rollout_name.upper() if rollout_name else "rollout"
+        print(f"Updated Prometheus configuration at {config.file} with {len(server_addresses)} {server_type} servers")
+
+        # Reload Prometheus on all nodes
+        reload_tasks = []
+        for node in alive_nodes:
+            node_ip = node["NodeManagerAddress"]
+            task = reload_prometheus.options(
+                resources={"node:" + node_ip: 0.001}  # Schedule to specific node
+            ).remote(config.port)
+            reload_tasks.append(task)
+
+        ray.get(reload_tasks)
+
+    except Exception as e:
+        logger.error(f"Failed to update Prometheus configuration: {e}")

code/RL_model/verl/verl_train/verl/experimental/agent_loop/single_turn_agent_loop.py

@@ -0,0 +1,84 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+from typing import Any
+from uuid import uuid4
+
+from verl.experimental.agent_loop.agent_loop import AgentLoopBase, AgentLoopOutput, register
+from verl.tools.utils.tool_registry import initialize_tools_from_config
+from verl.utils.profiler import simple_timer
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+@register("single_turn_agent")
+class SingleTurnAgentLoop(AgentLoopBase):
+    """Naive agent loop that only do single turn chat completion."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.prompt_length = self.config.actor_rollout_ref.rollout.prompt_length
+        self.response_length = self.config.actor_rollout_ref.rollout.response_length
+
+        tool_config_path = self.config.data.tool_config_path
+        tool_list = initialize_tools_from_config(tool_config_path) if tool_config_path else []
+        self.tool_schemas = [tool.tool_schema.model_dump(exclude_unset=True, exclude_none=True) for tool in tool_list]
+
+    async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput:
+        messages = list(kwargs["raw_prompt"])
+
+        # 1. extract images and videos from messages
+        multi_modal_data = await self.process_vision_info(messages)
+        images = multi_modal_data.get("images")
+        videos = multi_modal_data.get("videos")
+
+        # 2. apply chat template and tokenize
+        prompt_ids = await self.apply_chat_template(
+            messages,
+            tools=self.tool_schemas,
+            images=images,
+            videos=videos,
+        )
+
+        # 3. generate sequences
+        metrics = {}
+        with simple_timer("generate_sequences", metrics):
+            output = await self.server_manager.generate(
+                request_id=uuid4().hex,
+                prompt_ids=prompt_ids,
+                sampling_params=sampling_params,
+                image_data=images,
+                video_data=videos,
+            )
+        if metrics.get("num_preempted") is None:
+            metrics["num_preempted"] = output.num_preempted if output.num_preempted is not None else -1
+        response_mask = [1] * len(output.token_ids)
+
+        output = AgentLoopOutput(
+            prompt_ids=prompt_ids,
+            response_ids=output.token_ids[: self.response_length],
+            response_mask=response_mask[: self.response_length],
+            response_logprobs=output.log_probs[: self.response_length] if output.log_probs else None,
+            routed_experts=(
+                output.routed_experts[: len(prompt_ids) + self.response_length]
+                if output.routed_experts is not None
+                else None
+            ),
+            multi_modal_data=multi_modal_data,
+            num_turns=2,
+            metrics=metrics,
+        )
+        return output

code/RL_model/verl/verl_train/verl/experimental/agent_loop/tool_agent_loop.py

@@ -0,0 +1,475 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import json
+import logging
+import os
+from enum import Enum
+from typing import Any, Optional
+from uuid import uuid4
+
+import torch
+from PIL import Image
+from transformers import AutoProcessor, AutoTokenizer
+
+from verl.experimental.agent_loop.agent_loop import (
+    AgentLoopBase,
+    AgentLoopOutput,
+    AsyncLLMServerManager,
+    DictConfigWrap,
+    register,
+)
+from verl.experimental.agent_loop.tool_parser import FunctionCall, ToolParser
+from verl.experimental.agent_loop.utils import build_gpt_oss_tool_response_text
+from verl.interactions.base import BaseInteraction
+from verl.interactions.utils.interaction_registry import initialize_interactions_from_config
+from verl.tools.schemas import ToolResponse
+from verl.tools.utils.tool_registry import initialize_tools_from_config
+from verl.utils.profiler import simple_timer
+from verl.utils.rollout_trace import rollout_trace_op
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+class AgentState(Enum):
+    PENDING = "pending"
+    GENERATING = "generating"
+    PROCESSING_TOOLS = "processing_tools"
+    TERMINATED = "terminated"
+    INTERACTING = "interacting"
+
+
+class AgentData:
+    """Encapsulates all state variables for the agent loop. AgentData is passed to tool calling in case that
+    tool may need to access full history state. User can store any tool session data in `extra_fields`."""
+
+    def __init__(
+        self,
+        messages: list[dict[str, Any]],
+        image_data: list[Image.Image],
+        video_data: list[tuple[torch.Tensor, dict[str, Any]]],
+        metrics: dict[str, Any],
+        request_id: str,
+        tools_kwargs: dict[str, Any],
+        interaction: Optional[BaseInteraction] = None,
+        interaction_kwargs: Optional[dict[str, Any]] = None,
+    ):
+        self.messages = messages
+        self.image_data = image_data
+        self.video_data = video_data
+        self.metrics = metrics
+        self.request_id = request_id
+        self.tools_kwargs = tools_kwargs
+        self.interaction = interaction
+        self.interaction_kwargs = interaction_kwargs or {}
+
+        # State variables
+        self.prompt_ids: list[int] = []
+        self.response_ids: list[int] = []
+        self.response_mask: list[int] = []
+        self.response_logprobs: list[float] = []
+        self.turn_scores: list[float] = []
+        self.tool_rewards: list[float] = []
+        self.user_turns = 0
+        self.assistant_turns = 0
+
+        # Temporary state for tool calls
+        self.tool_calls: list[FunctionCall] = []
+
+        # Extra fields for dynamic addition, e.g., tool session data
+        self.extra_fields: dict[str, Any] = {}
+
+
+@register("tool_agent")
+class ToolAgentLoop(AgentLoopBase):
+    def __init__(
+        self,
+        trainer_config: DictConfigWrap,
+        server_manager: AsyncLLMServerManager,
+        tokenizer: AutoTokenizer,
+        processor: AutoProcessor,
+        **kwargs,
+    ):
+        super().__init__(trainer_config, server_manager, tokenizer, processor, **kwargs)
+        config = trainer_config.config
+
+        # Initialize tools from config file
+        self.max_user_turns = config.actor_rollout_ref.rollout.multi_turn.max_user_turns
+        self.max_assistant_turns = config.actor_rollout_ref.rollout.multi_turn.max_assistant_turns
+        self.max_parallel_calls = config.actor_rollout_ref.rollout.multi_turn.max_parallel_calls
+        self.max_tool_response_length = config.actor_rollout_ref.rollout.multi_turn.max_tool_response_length
+        self.tool_response_truncate_side = config.actor_rollout_ref.rollout.multi_turn.tool_response_truncate_side
+        tool_config_path = config.actor_rollout_ref.rollout.multi_turn.tool_config_path
+        tool_list = initialize_tools_from_config(tool_config_path) if tool_config_path else []
+        self.tools = {tool.name: tool for tool in tool_list}
+        self.tool_schemas = [tool.tool_schema.model_dump(exclude_unset=True, exclude_none=True) for tool in tool_list]
+        self.tool_parser = ToolParser.get_tool_parser(
+            config.actor_rollout_ref.rollout.multi_turn.format, self.tokenizer
+        )
+        self.tool_parser_name = config.actor_rollout_ref.rollout.multi_turn.format
+
+        self.prompt_length = config.actor_rollout_ref.rollout.prompt_length
+        self.response_length = config.actor_rollout_ref.rollout.response_length
+
+        # Initialize interactions from config file
+        self.interaction_config_file = config.actor_rollout_ref.rollout.multi_turn.interaction_config_path
+        if self.interaction_config_file:
+            self.interaction_map: dict[str, BaseInteraction] = self._initialize_interactions(
+                self.interaction_config_file
+            )
+
+    @rollout_trace_op
+    async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput:
+        messages = list(kwargs["raw_prompt"])
+
+        # extract images and videos from messages
+        multi_modal_data = await self.process_vision_info(messages)
+        images = multi_modal_data.get("images")
+        videos = multi_modal_data.get("videos")
+
+        metrics = {}
+        request_id = uuid4().hex
+        tools_kwargs = kwargs.get("tools_kwargs", {})
+
+        # Initialize interaction if needed
+        interaction = None
+        interaction_kwargs = {}
+        if self.interaction_config_file:
+            interaction_kwargs = kwargs["extra_info"]["interaction_kwargs"]
+            if "name" not in interaction_kwargs:
+                raise ValueError("'name' key is required in interaction_kwargs")
+            interaction_name = interaction_kwargs["name"]
+            if interaction_name not in self.interaction_map:
+                raise ValueError(
+                    f"Interaction '{interaction_name}' not found in interaction_map. Available interactions: "
+                    f"{list(self.interaction_map.keys())}"
+                )
+            interaction = self.interaction_map[interaction_name]
+            await interaction.start_interaction(request_id, **interaction_kwargs)
+        # Create AgentData instance to encapsulate all state
+        agent_data = AgentData(
+            messages=messages,
+            image_data=images,
+            video_data=videos,
+            metrics=metrics,
+            request_id=request_id,
+            tools_kwargs=tools_kwargs,
+            interaction=interaction,
+            interaction_kwargs=interaction_kwargs,
+        )
+
+        # State machine loop
+        state = AgentState.PENDING
+        while state != AgentState.TERMINATED:
+            if state == AgentState.PENDING:
+                state = await self._handle_pending_state(agent_data, sampling_params)
+            elif state == AgentState.GENERATING:
+                state = await self._handle_generating_state(agent_data, sampling_params)
+            elif state == AgentState.PROCESSING_TOOLS:
+                state = await self._handle_processing_tools_state(agent_data)
+            elif state == AgentState.INTERACTING:
+                state = await self._handle_interacting_state(agent_data)
+            else:
+                logger.error(f"Invalid state: {state}")
+                state = AgentState.TERMINATED
+
+        # Finalize output
+        response_ids = agent_data.prompt_ids[-len(agent_data.response_mask) :]
+        prompt_ids = agent_data.prompt_ids[: len(agent_data.prompt_ids) - len(agent_data.response_mask)]
+        multi_modal_data = {}
+        if agent_data.image_data is not None:
+            multi_modal_data["images"] = agent_data.image_data
+        if agent_data.video_data is not None:
+            multi_modal_data["videos"] = agent_data.video_data
+        output = AgentLoopOutput(
+            prompt_ids=prompt_ids,
+            response_ids=response_ids[: self.response_length],
+            response_mask=agent_data.response_mask[: self.response_length],
+            multi_modal_data=multi_modal_data,
+            response_logprobs=agent_data.response_logprobs[: self.response_length]
+            if agent_data.response_logprobs
+            else None,
+            num_turns=agent_data.user_turns + agent_data.assistant_turns + 1,
+            metrics=agent_data.metrics,
+            extra_fields={},
+        )
+        output.extra_fields.update({"turn_scores": agent_data.turn_scores, "tool_rewards": agent_data.tool_rewards})
+        return output
+
+    async def _handle_pending_state(self, agent_data: AgentData, sampling_params: dict[str, Any]) -> AgentState:
+        """Handle the pending state: prepare the prompt and start generation."""
+        prompt_ids = await self.apply_chat_template(
+            agent_data.messages,
+            tools=self.tool_schemas,
+            images=agent_data.image_data,
+            videos=agent_data.video_data,
+        )
+        agent_data.prompt_ids = prompt_ids
+        return AgentState.GENERATING
+
+    async def _handle_generating_state(
+        self, agent_data: AgentData, sampling_params: dict[str, Any], ignore_termination: bool = False
+    ) -> AgentState:
+        """Handle the generating state: generate model response and check for tool calls."""
+        add_messages: list[dict[str, Any]] = []
+
+        with simple_timer("generate_sequences", agent_data.metrics):
+            output = await self.server_manager.generate(
+                request_id=agent_data.request_id,
+                prompt_ids=agent_data.prompt_ids,
+                sampling_params=sampling_params,
+                image_data=agent_data.image_data,
+                video_data=agent_data.video_data,
+            )
+        # first time to set num_preempted
+        if agent_data.metrics.get("num_preempted") is None:
+            agent_data.metrics["num_preempted"] = output.num_preempted if output.num_preempted is not None else -1
+        # then add num_preempted to the metrics
+        else:
+            agent_data.metrics["num_preempted"] += output.num_preempted if output.num_preempted is not None else 0
+
+        agent_data.assistant_turns += 1
+        agent_data.response_ids = output.token_ids
+        agent_data.prompt_ids += agent_data.response_ids
+        agent_data.response_mask += [1] * len(agent_data.response_ids)
+        if output.log_probs:
+            agent_data.response_logprobs += output.log_probs
+
+        if output.routed_experts is not None:
+            agent_data.routed_experts = output.routed_experts
+
+        # Check termination conditions
+        if not ignore_termination and len(agent_data.response_mask) >= self.response_length:
+            return AgentState.TERMINATED
+        if self.max_assistant_turns and agent_data.assistant_turns >= self.max_assistant_turns:
+            return AgentState.TERMINATED
+        if self.max_user_turns and agent_data.user_turns >= self.max_user_turns:
+            return AgentState.TERMINATED
+
+        # Extract tool calls
+        _, agent_data.tool_calls = await self.tool_parser.extract_tool_calls(agent_data.response_ids)
+
+        # Handle interaction if needed
+        if self.interaction_config_file:
+            assistant_message = await self.loop.run_in_executor(
+                None, lambda: self.tokenizer.decode(agent_data.response_ids, skip_special_tokens=True)
+            )
+            add_messages.append({"role": "assistant", "content": assistant_message})
+            agent_data.messages.extend(add_messages)
+
+        # Determine next state
+        if agent_data.tool_calls:
+            return AgentState.PROCESSING_TOOLS
+        elif self.interaction_config_file:
+            return AgentState.INTERACTING
+        else:
+            return AgentState.TERMINATED
+
+    async def _handle_processing_tools_state(self, agent_data: AgentData) -> AgentState:
+        """Handle the processing tools state: execute tool calls and prepare tool responses."""
+        add_messages: list[dict[str, Any]] = []
+        new_images_this_turn: list[Any] = []  # Local variable instead of agent_data attribute
+
+        tasks = []
+        tool_call_names = []
+        for tool_call in agent_data.tool_calls[: self.max_parallel_calls]:
+            tasks.append(self._call_tool(tool_call, agent_data.tools_kwargs, agent_data))
+            tool_call_names.append(tool_call.name)
+
+        with simple_timer("tool_calls", agent_data.metrics):
+            responses = await asyncio.gather(*tasks)
+
+        # Process tool responses and update multi_modal_data
+        # Removed: agent_data.new_images_this_turn = []
+        for tool_response, tool_reward, _ in responses:
+            # Create message from tool response
+            if tool_response.image or tool_response.video:
+                # Multi-modal content with structured format
+                if not getattr(self.processor, "image_processor", None):
+                    raise ValueError(
+                        "Multimedia data can only be processed by `processor`, but the processor is None. "
+                        "This error is often caused if you are using a LLM model but your tool returns multimodal "
+                        "data. Plase use a vlm as the base model."
+                    )
+                content = []
+                if tool_response.image:
+                    content.append({"type": "image"})
+                if tool_response.video:
+                    content.append({"type": "video"})
+                if tool_response.text:
+                    content.append({"type": "text", "text": tool_response.text})
+                message = {"role": "tool", "content": content}
+            else:
+                # Text-only content
+                message = {"role": "tool", "content": tool_response.text or ""}
+
+            add_messages.append(message)
+
+            # Handle image data
+            if tool_response.image:
+                # Add new image data
+                if isinstance(tool_response.image, list):
+                    # Ensure all elements in the list are valid image objects
+                    for img in tool_response.image:
+                        if img is not None:  # Add a check to ensure the image is not None
+                            new_images_this_turn.append(img)  # Using local variable
+                else:
+                    # Ensure the image is not None
+                    if tool_response.image is not None:
+                        new_images_this_turn.append(tool_response.image)  # Using local variable
+
+            # Handle video data
+            if tool_response.video:
+                # Currently not supported, raise informative error
+                logger.warning("Multimedia type 'video' is not currently supported. Only 'image' is supported.")
+                raise NotImplementedError(
+                    "Multimedia type 'video' is not currently supported. Only 'image' is supported."
+                )
+
+            if tool_reward is not None:
+                agent_data.tool_rewards.append(tool_reward)
+
+        agent_data.messages.extend(add_messages)
+
+        if self.tool_parser_name == "gpt-oss":
+            logger.info("manually format tool responses for gpt-oss")
+            tool_response_text = build_gpt_oss_tool_response_text(add_messages, tool_call_names)
+            response_ids = await self.loop.run_in_executor(
+                None, lambda: self.tokenizer.encode(tool_response_text, add_special_tokens=False)
+            )
+        else:
+            response_ids = await self.apply_chat_template(
+                add_messages,
+                images=new_images_this_turn,  # Using local variable
+                videos=None,
+                remove_system_prompt=True,
+            )
+
+        if len(agent_data.response_mask) + len(response_ids) >= self.response_length:
+            return AgentState.TERMINATED
+        # Update prompt_ids and response_mask
+
+        if new_images_this_turn:
+            if agent_data.image_data is None:
+                agent_data.image_data = []
+            elif not isinstance(agent_data.image_data, list):
+                agent_data.image_data = [agent_data.image_data]
+            for img in new_images_this_turn:
+                agent_data.image_data.append(img)
+
+        agent_data.prompt_ids += response_ids
+        agent_data.response_mask += [0] * len(response_ids)
+        if agent_data.response_logprobs:
+            agent_data.response_logprobs += [0.0] * len(response_ids)
+        agent_data.user_turns += 1
+        return AgentState.GENERATING
+
+    async def _handle_interacting_state(self, agent_data: AgentData) -> AgentState:
+        """Handle the interacting state: get user input from interaction."""
+        (
+            should_terminate_sequence,
+            interaction_responses,
+            reward,
+            metrics,
+        ) = await agent_data.interaction.generate_response(
+            agent_data.request_id, agent_data.messages, **agent_data.interaction_kwargs
+        )
+        agent_data.user_turns += 1
+
+        add_messages: list[dict[str, Any]] = [{"role": "user", "content": interaction_responses}]
+        agent_data.messages.extend(add_messages)
+
+        if reward is not None:
+            agent_data.turn_scores.append(reward)
+
+        # Update prompt with user responses (similar to _handle_processing_tools_state)
+        response_ids = await self.apply_chat_template(
+            add_messages,
+            remove_system_prompt=True,
+        )
+
+        # Update prompt_ids and response_mask
+        agent_data.prompt_ids += response_ids
+        agent_data.response_mask += [0] * len(response_ids)
+        if agent_data.response_logprobs:
+            agent_data.response_logprobs += [0.0] * len(response_ids)
+
+        # double check prompt
+        # Check termination condition
+        if should_terminate_sequence:
+            return AgentState.TERMINATED
+        else:
+            return AgentState.GENERATING
+
+    async def _call_tool(
+        self, tool_call: FunctionCall, tools_kwargs: dict[str, Any], agent_data: AgentData
+    ) -> tuple[ToolResponse, float, dict]:
+        """Call tool and return tool response."""
+        tool, instance_id = None, None
+        try:
+            # TODO: append malformed tool_call to the prompt: invalid function name or arguments
+            tool_name = tool_call.name
+            tool_args = json.loads(tool_call.arguments)
+            tool = self.tools[tool_name]
+            kwargs = tools_kwargs.get(tool_name, {})
+            instance_id, _ = await tool.create(create_kwargs=kwargs.get("create_kwargs", {}))
+            tool_execution_response, tool_reward, res = await tool.execute(
+                instance_id, tool_args, agent_data=agent_data
+            )
+        except Exception as e:
+            logger.warning(f"Error when executing tool: {e}")
+            return (
+                ToolResponse(
+                    text=f"Error when executing tool: {e}",
+                ),
+                0.0,
+                {},
+            )
+        finally:
+            if tool and instance_id:
+                await tool.release(instance_id)
+
+        tool_response_text = tool_execution_response.text
+        if tool_response_text and len(tool_response_text) > self.max_tool_response_length:
+            if self.tool_response_truncate_side == "left":
+                tool_response_text = tool_response_text[: self.max_tool_response_length] + "...(truncated)"
+            elif self.tool_response_truncate_side == "right":
+                tool_response_text = "(truncated)..." + tool_response_text[-self.max_tool_response_length :]
+            else:
+                length = self.max_tool_response_length // 2
+                tool_response_text = tool_response_text[:length] + "...(truncated)..." + tool_response_text[-length:]
+
+        # Create ToolResponse from tool execution result
+        tool_response_kwargs = {"text": tool_response_text}
+
+        # Add multimedia data if present
+        for attr_name in ["image", "video"]:
+            if hasattr(tool_execution_response, attr_name):
+                attr_value = getattr(tool_execution_response, attr_name)
+                if attr_value is not None:
+                    tool_response_kwargs[attr_name] = attr_value
+
+        return ToolResponse(**tool_response_kwargs), tool_reward, res
+
+    def _initialize_interactions(self, interaction_config_file):
+        """Initialize interactions from configuration.
+        Returns:
+            dict[str, BaseInteraction]: A dictionary mapping interaction names to interaction instances.
+        """
+        if interaction_config_file is None:
+            return {}
+
+        interaction_map = initialize_interactions_from_config(interaction_config_file)
+        return interaction_map

code/RL_model/verl/verl_train/verl/experimental/agent_loop/tool_parser.py

@@ -0,0 +1,161 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+
+import regex
+from pydantic import BaseModel
+
+from verl.utils.ray_utils import get_event_loop
+from verl.utils.rollout_trace import rollout_trace_op
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+class FunctionCall(BaseModel):
+    arguments: str
+    """
+    The arguments to call the function with, as generated by the model in JSON
+    format. Note that the model does not always generate valid JSON, and may
+    hallucinate parameters not defined by your function schema. Validate the
+    arguments in your code before calling your function.
+    """
+
+    name: str
+    """The name of the function to call."""
+
+
+class ToolParser(ABC):
+    _registry: dict[str, type["ToolParser"]] = {}
+
+    def __init__(self, tokenizer) -> None:
+        self.tokenizer = tokenizer
+
+    @abstractmethod
+    async def extract_tool_calls(self, responses_ids: list[int]) -> tuple[str, list[FunctionCall]]:
+        """Extract tool calls from the responses.
+
+        Args:
+            responses_ids (List[int]): The ids of the responses.
+
+        Returns:
+            Tuple[str, List[FunctionCall]]: Content and extracted tool calls.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def get_tool_parser(cls, name: str, tokenizer):
+        if name not in cls._registry:
+            raise ValueError(f"Unknown tool parser: {name}")
+        return cls._registry[name](tokenizer)
+
+    @classmethod
+    def register(cls, name: str):
+        def decorator(subclass: type[ToolParser]) -> type[ToolParser]:
+            cls._registry[name] = subclass
+            return subclass
+
+        return decorator
+
+
+@ToolParser.register("hermes")
+class HermesToolParser(ToolParser):
+    """Adapted from https://github.com/vllm-project/vllm/blob/v0.9.1/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py"""
+
+    def __init__(self, tokenizer) -> None:
+        super().__init__(tokenizer)
+
+        self.tool_call_start_token: str = "<tool_call>"
+        self.tool_call_end_token: str = "</tool_call>"
+        self.tool_call_regex = regex.compile(r"<tool_call>(.*?)</tool_call>", regex.DOTALL)
+
+    @rollout_trace_op
+    async def extract_tool_calls(self, responses_ids: list[int]) -> tuple[str, list[FunctionCall]]:
+        loop = get_event_loop()
+        text = await loop.run_in_executor(None, self.tokenizer.decode, responses_ids)
+        if self.tool_call_start_token not in text or self.tool_call_end_token not in text:
+            return text, []
+
+        matches = self.tool_call_regex.findall(text)
+        function_calls = []
+        for match in matches:
+            try:
+                function_call = json.loads(match)
+                name, arguments = function_call["name"], function_call["arguments"]
+                function_calls.append(FunctionCall(name=name, arguments=json.dumps(arguments, ensure_ascii=False)))
+            except Exception as e:
+                logger.error(f"Failed to decode tool call: {e}")
+
+        # remaing text exclude tool call tokens
+        content = self.tool_call_regex.sub("", text)
+
+        return content, function_calls
+
+
+@ToolParser.register("gpt-oss")
+class GptOssToolParser(ToolParser):
+    """
+    Tool parser for gpt-oss model.
+    Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/function_call/gpt_oss_detector.py
+
+    Args:
+        tokenizer: The tokenizer to use.
+    """
+
+    def __init__(self, tokenizer) -> None:
+        super().__init__(tokenizer)
+        # check https://cookbook.openai.com/articles/openai-harmony for more details.
+        self.cot_pattern = regex.compile(
+            r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>.*?<\|end\|>", regex.DOTALL
+        )
+        # <|start|>assistant may be pre-appended in prompts, so we need to remove it.
+        self.partial_cot_pattern = regex.compile(r"<\|channel\|>analysis<\|message\|>(.*?)<\|end\|>", regex.DOTALL)
+        self.tool_call_pattern = regex.compile(
+            r"<\|start\|>assistant<\|channel\|>[^<]* to=functions\.([^<]+) "
+            r"<\|constrain\|>json<\|message\|>(.*?)<\|call\|>",
+            regex.DOTALL,
+        )
+
+    @rollout_trace_op
+    async def extract_tool_calls(self, responses_ids: list[int]) -> tuple[str, list[FunctionCall]]:
+        loop = get_event_loop()
+        # We need to keep special tokens for gpt-oss model for better tool call extraction.
+        text = await loop.run_in_executor(None, lambda: self.tokenizer.decode(responses_ids, skip_special_tokens=False))
+        # Need to remove padding tokens for better tool call extraction.
+        text = text.replace(self.tokenizer.pad_token, "")
+        # Need to reomve COT since COT may contain tool call tokens.But they are not valid tool calls.
+        text = regex.sub(self.cot_pattern, "", text)
+        text = regex.sub(self.partial_cot_pattern, "", text)
+
+        # check if there are tool calls in the text by re.findall
+        matches = regex.findall(self.tool_call_pattern, text)
+        if not matches:
+            return text, []
+
+        function_calls = []
+        for match in matches:
+            try:
+                name, arguments = match[0], match[1]
+                # don't check if arguments is valid JSON and leave it to client
+                function_calls.append(FunctionCall(name=name, arguments=arguments))
+            except Exception as e:
+                logger.error(f"Failed to decode tool call: {e}")
+
+        # remaing text exclude tool call tokens
+        content = regex.sub(self.tool_call_pattern, "", text)
+
+        return content, function_calls

code/RL_model/verl/verl_train/verl/experimental/agent_loop/utils.py

@@ -0,0 +1,108 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Any
+
+
+def resolve_config_path(config_path: str) -> str:
+    """Resolve agent loop configuration file path.
+
+    In multi-node Ray training, relative paths may not resolve correctly
+    because the working directory on remote nodes can differ from the driver node.
+    This function resolves relative paths by checking multiple locations in order:
+    1. If already absolute, return as-is
+    2. Try current working directory
+    3. Try relative to verl package installation (project root)
+
+    Args:
+        config_path: Configuration file path (relative or absolute)
+
+    Returns:
+        Absolute path to the configuration file
+
+    Raises:
+        FileNotFoundError: If the configuration file cannot be found
+    """
+    # Return absolute paths unchanged
+    if os.path.isabs(config_path):
+        return config_path
+
+    # Try current working directory first
+    cwd = os.path.abspath(os.getcwd())
+    cwd_path = os.path.abspath(os.path.join(cwd, config_path))
+    if (cwd_path == cwd or cwd_path.startswith(cwd + os.sep)) and os.path.exists(cwd_path):
+        return cwd_path
+
+    # Try relative to verl project root (where verl package is installed)
+    try:
+        import verl
+
+        verl_package_dir = os.path.abspath(os.path.dirname(verl.__file__))
+
+        # Strategy 1: For development/editable installs.
+        project_root = os.path.dirname(verl_package_dir)
+        dev_path = os.path.abspath(os.path.join(project_root, config_path))
+        if (dev_path == project_root or dev_path.startswith(project_root + os.sep)) and os.path.exists(dev_path):
+            return dev_path
+
+        # Strategy 2: For standard package installations.
+        install_path = os.path.abspath(os.path.join(verl_package_dir, config_path))
+        if (install_path == verl_package_dir or install_path.startswith(verl_package_dir + os.sep)) and os.path.exists(
+            install_path
+        ):
+            return install_path
+    except (ImportError, AttributeError):
+        pass  # verl not installed or __file__ not available
+
+    # File not found - raise clear error
+    raise FileNotFoundError(
+        f"Agent loop configuration file not found: {config_path}. Tried current directory and verl project root."
+    )
+
+
+# tokenizer.apply_chat_template is not working properly for gpt-oss model.
+# Because the chat template requires tool call messages to parse tool response messages
+# so we need to format the tool response manually.
+def format_gpt_oss_tool_response_manually(tool_response: str, tool_call_name: str) -> str:
+    """Format tool response for gpt-oss model.
+    Args:
+        tool_response: Tool response string
+        tool_call_name: Name of the tool that was called
+
+    Returns:
+        Formatted tool response string
+    """
+    return f"<|start|>functions.{tool_call_name} to=assistant<|channel|>commentary<|message|>{tool_response}<|end|>"
+
+
+def add_generation_prompt_for_gpt_oss(message_content: str) -> str:
+    """Add generation prompt for gpt-oss model.
+    Args:
+        message_content: Message content string
+
+    Returns:
+        Message content string with generation prompt
+    """
+    return message_content + "<|start|>assistant"
+
+
+def build_gpt_oss_tool_response_text(messages: list[dict[str, Any]], tool_call_names: list[str]) -> str:
+    """Build gpt-oss tool response text (manual formatting + generation prompt)."""
+    tool_response_texts: list[str] = []
+    for i, tool_msg in enumerate(messages):
+        actual_tool_name = tool_call_names[i]
+        formatted = format_gpt_oss_tool_response_manually(tool_msg["content"], actual_tool_name)
+        tool_response_texts.append(formatted)
+    return add_generation_prompt_for_gpt_oss("".join(tool_response_texts))

code/RL_model/verl/verl_train/verl/experimental/dataset/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

code/RL_model/verl/verl_train/verl/experimental/dataset/sampler.py

@@ -0,0 +1,40 @@
+# Copyright 2025 Amazon.com Inc and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from abc import abstractmethod
+from collections.abc import Sized
+
+from omegaconf import DictConfig
+from torch.utils.data import Sampler
+
+from verl import DataProto
+
+
+class AbstractSampler(Sampler[int]):
+    """Abstract interface for custom samplers."""
+
+    @abstractmethod
+    def __init__(
+        self,
+        data_source: Sized,
+        data_config: DictConfig,
+    ):
+        pass
+
+
+class AbstractCurriculumSampler(AbstractSampler):
+    """Experimental interface for curriculum learning samplers."""
+
+    @abstractmethod
+    def update(self, batch: DataProto) -> None:
+        pass

code/RL_model/verl/verl_train/verl/experimental/dynamic_dataset/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

code/RL_model/verl/verl_train/verl/experimental/dynamic_dataset/dynamicgen_dataset.py

@@ -0,0 +1,112 @@
+# Copyright 2025 Amazon.com Inc and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Dataset class that enables dynamic data generation strategies between iterations of training.
+This class extends RLHFDataset and uses an AbstractDataGen instance to generate data.
+
+This is especially useful in settings where proposer model generates new tasks based
+on rollout data.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import datasets
+from omegaconf import DictConfig
+from torch.utils.data import Dataset
+from transformers import PreTrainedTokenizer, ProcessorMixin
+
+from verl import DataProto
+from verl.utils.dataset import RLHFDataset
+from verl.utils.import_utils import load_extern_object
+
+logger = logging.getLogger(__name__)
+
+
+class AbstractDataGenerator(ABC):
+    def __init__(self, config: DictConfig):
+        self.config = config
+
+    @abstractmethod
+    def generate(self, dataset: Dataset) -> datasets.Dataset:
+        """
+        Generate method must be implemented by subclasses.
+        Args:
+            dataset: The dataset to generate from.
+        Returns:
+            Processed data or result as implemented by the subclass.
+        """
+        pass
+
+
+class MockDataGenerator(AbstractDataGenerator):
+    """
+    A noop data gen class that only reappends the first datapoint.
+    This class is useful as a placeholder and testing.
+    """
+
+    def __init__(self, config: DictConfig = None):
+        super().__init__(config)
+
+    def generate(self, dataset: Dataset) -> datasets.Dataset:
+        print("MockDataGenerator: No operation performed on the dataset.")
+        return dataset.dataframe.select([0])
+
+
+class DynamicGenDataset(RLHFDataset):
+    """
+    A dataset class that uses a data generation strategy to process data.
+    This class extends RLHFDataset and uses an AbstractDataGen instance to generate data.
+    """
+
+    def __init__(
+        self,
+        data_files: str | list[str],
+        tokenizer: PreTrainedTokenizer,
+        config: DictConfig,
+        processor: Optional[ProcessorMixin] = None,
+    ):
+        super().__init__(data_files, tokenizer, config, processor)
+        self.datagen: AbstractDataGenerator = config.datagen
+        assert "datagen" in config and config.datagen.get("path", None) is not None, (
+            f"datagen path is not set in config: {config}"
+        )
+        # Dynamically load the custom datagen class
+        datagen_cls = load_extern_object(config.datagen.path, config.datagen.name)
+
+        # Verify that the custom datagen class inherits from AbstractDataGenerator
+        abs_cls = AbstractDataGenerator
+        if not issubclass(datagen_cls, abs_cls):
+            raise TypeError(
+                f"The custom datagen class '{config.datagen.name}' from '{config.datagen.path}'"
+                + " must inherit from {abs_cls}"
+            )
+
+        self.data_generator = datagen_cls(config.datagen)
+        self.on_batch_end()
+
+    def append_dataframe(self, new_dataframe: datasets.Dataset):
+        new_dataframe = self.maybe_filter_out_long_prompts(new_dataframe)
+        self.dataframe = datasets.concatenate_datasets([self.dataframe, new_dataframe])
+
+        logger.info(f"new dataset len: {len(self.dataframe)}")
+
+    def on_batch_end(self, batch: DataProto) -> None:
+        """
+        Generate data using the provided data generation strategy.
+        Note: This method is intended to change the dataset after each training batch.
+        """
+        new_data = self.data_generator.generate(self)
+        self.append_dataframe(new_data)

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/README.md

@@ -0,0 +1,599 @@
+# Recipe: Fully Async Policy Trainer
+
+**Author:** `https://github.com/meituan-search`
+
+Last updated: 12/25/2025.
+
+This document introduces a fully asynchronous PPO training system that completely decouples the Trainer and Rollouter,
+supporting asynchronous sample generation and training.
+Under this system, we achieved a 2.35x-2.67x performance improvement when training the Qwen2.5-7B model with 128 GPUs,
+without significantly affecting the results.
+
+## Introduction
+
+### Background
+
+The separated rollout and train architecture, compared to the colocate architecture, can allocate resources more
+flexibly and design more flexible training logic, thereby addressing issues such as low GPU utilization and training
+efficiency caused by long-tail problems.
+The one_step_off_policy alleviates the problem of long rollout times and achieves some gains in training efficiency by
+designing a separated architecture and performing asynchronous training between rollout and train for one round.
+However, it forcibly uses data from one round of asynchronous training, which is not flexible enough and cannot
+completely eliminate the impact of long-tail on training efficiency.
+In other frameworks such as AReaL, Magistral, StreamRL, and AsyncFlow, asynchronous training and streaming training have
+been implemented based on the separated architecture and have achieved gains.
+We borrow from their methods and implemented them in VERL. The fully_async_policy supports asynchronous, streaming, and
+partial
+rollout training.
+By reasonably setting parameters such as resource allocation and parameter synchronization frequency, fully_async_policy
+can significantly improve training efficiency.
+
+> Magistral https://arxiv.org/abs/2506.10910
+>
+> AReaL: A Large-Scale Asynchronous Reinforcement Learning System for Language
+> Reasoning https://arxiv.org/abs/2505.24298
+>
+> StreamRL: Scalable, Heterogeneous, and Elastic RL for LLMs with Disaggregated Stream
+> Generation https://arxiv.org/abs/2504.15930
+>
+> AsyncFlow: An Asynchronous Streaming RL Framework for Efficient LLM Post-Training https://arxiv.org/abs/2507.01663
+>
+
+### Core Contributions
+
+* **Resource Isolation**: Unlike using hybrid_engine, Rollouter and Trainer use separate computing resources and need to
+  specify the resources they occupy separately.
+* **Parallel Generation and Training**: While the Trainer is training, the Rollouter is generating new samples.
+* **Multi-step Asynchronous**: Compared to one step off policy, it supports asynchronous settings from 0.x steps to
+  multiple steps, making the asynchronous solution more flexible.
+* **NCCL Parameter Synchronization**: Based on the nccl communication primitive, refer to [checkpoint-engine](https://github.com/MoonshotAI/checkpoint-engine) to
+  achieve efficient parameter synchronization between Rollouter and Trainer.
+* **Stream Inference and Training**: Rollouter generates data sample by sample, and data transmission uses a single
+  sample as the minimum transmission unit.
+* **Asynchronous Training and Freshness Control**: By setting the parameter async_training.staleness_threshold, it
+  supports training with samples generated by old parameters.
+* **PartialRollout**: The Rollouter's inference process supports partial rollout logic. During parameter
+  synchronization, by adding `sleep() and resume()` logic, it
+  saves samples from ongoing rollouts and continues using them in the next rollout, reducing the time spent waiting for
+  ongoing tasks to finish during parameter synchronization.
+
+Currently, the supported usage mode is megatron/fsdp+vllm. vllm must use the server mode based on AgentLoop.
+
+## Design
+
+The overall architecture of fully_async_policy is shown in the figure below. fully_async_policy mainly consists of four
+parts: Rollouter, MessageQueue, Trainer, and ParameterSynchronizer.
+
+![fully_async_policy_structure](
+https://github.com/ArronHZG/verl-community/blob/main/docs/fully_async_policy_structure.svg?raw=true)
+
+1. Rollouter generates sequences sample by sample and puts the generated samples into the MessageQueue, with the
+   production speed controlled by freshness.
+2. MessageQueue is used to temporarily store samples generated by Rollouter.
+3. Trainer fetches samples from MessageQueue sample by sample. After fetching `require_batches*ppo_mini_batch_size`
+   samples, it will perform training. After training for async_training.trigger_parameter_sync_step rounds, it triggers
+   a parameter synchronization with Rollouter.
+4. ParameterSynchronizer implements the NCCL synchronous parameter synchronization capability.
+
+The source of benefits compared to the base scheme lies in the fact that in the colocate case, using more resources for
+rollout cannot solve the idleness caused by long-tail samples.
+After we perform resource isolation, the time for rollout and train may be longer than before (because fewer resources
+are used),
+but the overlap in their time consumption reduces the end-to-end time consumption.
+
+![fully_async_policy_revenue](
+https://github.com/ArronHZG/verl-community/blob/main/docs/fully_async_policy_revenue.svg?raw=true)
+
+## Usage
+
+### Parameter Description
+
+| super params                                  | implication                                                                                    |
+|-----------------------------------------------|------------------------------------------------------------------------------------------------|
+| `trainer.nnodes`                              | Number of nodes for Trainer                                                                    |
+| `trainer.n_gpus_per_node`                     | Number of GPUs per node for Trainer                                                            |
+| `rollout.nnodes`                              | Number of nodes for Rollouter                                                                  |
+| `rollout.n_gpus_per_node`                     | Number of GPUs per node for Rollouter                                                          |
+| `data.train_batch_size`                       | In the fully async strategy, this value is not effective (default is 0)                        |
+| `data.gen_batch_size`                         | In the fully async strategy, uses streaming sample production logic (default is 1)             |
+| `rollout.total_rollout_steps`                 | Total number of rollout samples                                                                |
+| `rollout.test_freq`                           | How many times Rollouter updates parameters before performing a validation                     |
+| `actor_rollout_ref.actor.ppo_mini_batch_size` | The ppo_mini_batch_size is a global num across all workers/gpus                                |
+| `async_training.require_batches`              | Number of ppo_mini_batch_size that FullyAsyncTrainer fetches at once                           |
+| `async_training.trigger_parameter_sync_step`  | Indicates how many local updates FullyAsyncTrainer performs before a parameter synchronization |
+| `async_training.staleness_threshold`          | Freshness control                                                                              |
+| `async_training.partial_rollout`              | Whether to perform partial_rollout                                                             |
+| `async_training.use_rollout_log_probs`        | Use log_probs generated by rollout                                                             |
+| `async_training.compute_prox_log_prob`        | Whether to compute log_prob using the training model's parameters during the training phase.   |                                                |
+| `async_training.checkpoint_engine.enable`| Whether to use checkpoint_engine for accelerating, default `True`|
+| `async_training.checkpoint_engine.overlap_broadcast_and_consume` | When use checkpoint_engine, whether to overlap broadcast and load_weights, default `False`|
+| `async_training.checkpoint_engine.device_buffer_size_M` | When use checkpoint_engine, the user-specific bucket size (MB), default `4096`|
+| `async_training.use_trainer_do_validate` | Whether use trainer node to do validate process, default `False`|
+
+**Further Explanation:**
+
+* `rollout.total_rollout_steps`
+
+  Compared to colocate, the quantity can be aligned by multiplying train_batch_size and step:
+  `rollout.total_rollout_steps = data.train_batch_size * step`.
+
+* `async_training.trigger_parameter_sync_step`
+
+  In the fully async strategy, it indicates how many local updates the Trainer performs (i.e., how many times it fetches
+  `require_batches * ppo_mini_batch_size` samples) before a parameter synchronization with Rollouter.
+  Between every two parameter synchronizations between Rollouter and Trainer, the Trainer will process
+  `trigger_parameter_sync_step* require_batches*ppo_mini_batch_size` samples.
+  To fairly compare speed with colocate, `trigger_parameter_sync_step` should be set to
+  `data.train_batch_size / (require_batches * ppo_mini_batch_size)`.
+
+* `async_training.staleness_threshold`
+
+  In the fully async strategy, it indicates the maximum proportion of stale samples allowed to be used.
+
+    * `staleness_threshold`=0, indicates synchronous training.
+      Rollouter will generate a fixed number of samples between two parameter updates, the sample count is:
+      
+      `rollout_num = (trigger_parameter_sync_step*require_batches*ppo_mini_batch_size)`
+    * `staleness_threshold`>0, indicates asynchronous training, can be set to a decimal for more flexible asynchronous
+      calls.
+      Rollouter will generate at most the following number of samples between two parameter updates:
+
+      `rollout_num = (1+staleness_threshold)*(trigger_parameter_sync_step*require_batches*ppo_mini_batch_size) - num_staleness_sample`
+
+  `num_staleness_sample` represents the number of stale samples generated in excess during the last rollout.
+
+  Since it's a streaming system, rollout continues to generate and trainer continues to consume. If rollouter is slower,
+  trainer will trigger parameter synchronization earlier, and rollouter will not actually produce rollout_num samples.
+  When rollout is fast enough, setting `staleness_threshold` to 1 is basically equivalent to one_step_off policy.
+  To avoid too many expired samples affecting training accuracy, it is recommended to set this value to less than 1.
+
+* `async_training.partial_rollout`
+
+  partial_rollout only actually takes effect when staleness_threshold>0.
+
+* `async_training.use_rollout_log_probs`
+
+  In reinforcement learning algorithms, log_probs have implicit correlations with parameter versions and tokens. Due to
+  the settings of algorithms like PPO/GRPO/DAPO, when calculating importance sampling,
+  old_log_prob must use the log_probs corresponding to the rollout parameters and tokens to ensure algorithm
+  correctness. In the fully
+  async strategy, we default to old_log_prob being calculated by rollout rather than by trainer.
+
+* `async_training.require_batches`
+
+  In streaming training, require_batches should be set to 1, indicating that training is performed after producing
+  enough ppo_mini_batch_size samples.
+  In actual testing, we found that if fewer samples are issued at once, due to the order of data distribution, it can
+  cause training instability and longer response lengths.
+  Here, we additionally provide require_batches for streaming distribution and control the number of samples
+  participating in training at once.
+
+* `async_training.compute_prox_log_prob` (experimental)
+
+  During the training process, we observed that metrics and response lengths may become unstable in the later
+  stages of training. To mitigate this issue, we can use
+  the [Rollout Importance Sampling](https://verl.readthedocs.io/en/latest/advance/rollout_is.html)
+  technique for importance sampling. To utilize Rollout Importance Sampling, we need to compute log_prob using
+  the training engine, which requires enabling this switch.
+  Additionally, when compute_prox_log_prob and Rollout Importance Sampling are enabled under mode d
+  (async stream pipeline with partial rollout), our implementation approximates `Areal's Decoupled PPO`.
+
+* `async_training.checkpoint_engine.enable`
+  
+  Enabling the checkpoint engine generally reduces synchronization time overhead by more than 60% compared to 
+  the original per-tensor parameter synchronization method. However, assembling buckets incurs additional 
+  temporary GPU memory overhead.
+
+* `async_training.checkpoint_engine.overlap_broadcast_and_consume`
+
+  Enabling pipeline between the broadcast and load_weights parameters will allocate additional GPU memory. 
+  Since the main time consumption for parameter synchronization is not in the broadcast and load_weights phases,
+  but in the parameter generation phase (by megatron or FSDP), this option is off by default.
+
+* `async_training.checkpoint_engine.device_buffer_size_M`
+  
+  It controls the size of the memory buffer used for synchronization when the checkpoint-engine is enabled. 
+  The actual `bucket_size` = `max(device_buffer_size_M, maximum parameter tensor size)`.
+  * When enable `overlap_broadcast_and_consume`, the additional device memory overhead of 
+    trainer rank is `3 * bucket_size`and rollout rank is `2 * bucket_size`。
+  * When disable `overlap_broadcast_and_consume`, the additional device memory overhead of 
+    trainer rank is `2 * bucket_size`and rollout rank is `1 * bucket_size`。
+
+* `async_training.use_trainer_do_validate`
+
+  It controls whether to use the trainer's `do_validate` method for validation.
+  If set to True, the trainer will perform validation after each parameter update. It can reduce the validation time
+  overhead and trainer node idle time.
+  If set to False, the trainer will not perform validation.
+
+### Supported Modes
+
+1. on policy pipeline:
+    1. **trigger_parameter_sync_step=1, staleness_threshold=0**
+    2. Rollouter produces `require_batches*ppo_mini_batch_size` samples at once, Trainer fetches these samples for
+       training, and after training completes, Trainer and Rollouter perform a parameter synchronization;
+    3. During the rollout phase, if there are long-tail samples but few rollout samples, shorter samples cannot fill
+       idle resources, causing some resource waste.
+    4. As shown in figure a;
+
+2. stream off policy pipeline:
+    1. **trigger_parameter_sync_step>1, staleness_threshold=0**
+    2. Synchronous streaming training will be performed. Rollouter produces
+       `require_batches*ppo_mini_batch_size*trigger_parameter_sync_step` samples at once, Trainer performs a local
+       training every time it fetches `require_batches*ppo_mini_batch_size` samples, and after training
+       trigger_parameter_sync_step times, Trainer and Rollouter perform a parameter synchronization;
+    3. Compared to a, since more samples are generated at once, resource idleness will be lower.
+    4. In one step training, there will be two periods of resource idleness: when fetching the first batch of samples,
+       train waits for `require_batches*ppo_mini_batch_size` samples to be produced, and during the last parameter
+       update, rollout waits for training to complete.
+    5. As shown in figure b;
+
+3. async stream pipeline with stale samples:
+    1. **trigger_parameter_sync_step>=1, staleness_threshold>0, partial_rollout=False**
+    2. After each parameter update, Rollouter will plan to produce at most rollout_num samples (in practice, the number
+       of samples generated may be less than this value depending on rollout speed).
+    3. If the rollout process is relatively fast, Rollouter will generate some additional samples num_stale_samples
+       before parameter synchronization for immediate use by Trainer after synchronization.
+       When triggering parameter synchronization, if Rollouter has ongoing tasks, it will wait for the tasks to complete
+       and not add new tasks;
+    4. Compared to b, except for the first step training, subsequent training will not have the time to wait for the
+       first batch rollout to finish, but will have the time to wait for active tasks to finish.
+    5. As shown in figure c;
+
+4. async stream pipeline with partial rollout:
+    1. **trigger_parameter_sync_step>=1, staleness_threshold>0, partial_rollout=True**
+    2. Compared to c, when triggering parameter synchronization, if Rollouter has samples being produced, it will
+       interrupt the rollout process and perform parameter synchronization. The interrupted samples will continue to be
+       generated after synchronization. This reduces the time to wait for active tasks to finish.
+    3. As shown in figure d;
+
+![fully_async_policy_mode](
+https://github.com/ArronHZG/verl-community/blob/main/docs/fully_async_policy_mode.svg?raw=true)
+
+### Key Metrics
+
+| metrics                                        | implication                                                                                            |
+|------------------------------------------------|--------------------------------------------------------------------------------------------------------|
+| `trainer/idle_ratio`                           | Trainer idle rate                                                                                      |
+| `rollouter/idle_ratio`                         | Rollouter idle rate                                                                                    |
+| `fully_async/count/stale_samples_processed`    | Total number of old samples used in training                                                           |
+| `fully_async/count/stale_trajectory_processed` | Total number of old trajectories used in training (one sample produces rollout.n trajectories)         |
+| `fully_async/partial/total_partial_num`        | Number of partial samples processed by Trainer between two trigger_parameter_sync_step                 |
+| `fully_async/partial/partial_ratio`            | Ratio of partial samples processed by Trainer between two trigger_parameter_sync_step                  |
+| `fully_async/partial/max_partial_span`         | Maximum parameter span of partial samples processed by Trainer between two trigger_parameter_sync_step |
+
+### Parameter Tuning Recommendations
+
+* Resource Allocation and Adjustment:
+    * Reasonable resource allocation is the prerequisite for achieving good training efficiency. The ideal resource
+      allocation should make the rollout time and train time close, thereby minimizing pipeline bubbles in the entire
+      training process,
+      avoiding resource idleness, and ensuring Trainer does not use old samples. In real training scenarios, resource
+      allocation can be adjusted based on the idle time of rollout and train during actual training,
+      which can be obtained from rollouter/idle_ratio and trainer/idle_ratio. If rollouter/idle_ratio is high and
+      trainer/idle_ratio is low,
+      Trainer resources should be increased and Rollouter resources should be reduced, and vice versa.
+
+* Key Parameters:
+    * staleness_threshold: Setting it too high will cause more old samples to be used, affecting model performance. It
+      is recommended to set it to less than 1.
+    * require_batches: The closer to 1, the closer to a pure streaming process, the smaller the training bubbles, and
+      the faster the acceleration effect that can be achieved in terms of speed, but it will affect the order of sample
+      processing;
+    * trigger_parameter_sync_step: The smaller the setting, the closer to on policy, but it will cause frequent
+      parameter synchronization. Long-tail samples waste resources that cannot be filled by short samples, resulting in
+      low resource utilization.
+      The larger the setting, the higher the computational efficiency, but the accuracy will be affected by off policy.
+    * rollout.test_freq: It will occupy Rollouter resources and is not recommended to be set too small.
+
+* Mode Selection: By adjusting different parameters, the Fully Async architecture supports optimization acceleration at
+  different levels, suitable for tasks in different scenarios.
+    * For small-scale tasks that need to ensure training stability and on-policy nature, and have low speed
+      requirements, the on policy pipeline mode (Mode 1) can be tried.
+    * For scenarios that need to improve training throughput but are sensitive to staleness, the stream off policy
+      pipeline mode can be tried. That is, by
+      setting trigger_parameter_sync_step>1 to improve training efficiency, but still maintaining the synchronization
+      mechanism (staleness_threshold=0) (Mode 2).
+    * For large-scale tasks with high training speed requirements and can tolerate a certain degree of off-policy and
+      staleness, setting staleness_threshold>
+      0 and partial_rollout=True can improve training efficiency, using the async stream pipeline mode (Mode 3 or 4).
+
+### Quick Start
+
+```shell
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*400)))
+test_freq=10
+staleness_threshold=0
+trigger_parameter_sync_step=16
+partial_rollout=False
+
+
+python -m recipe.fully_async_policy.fully_async_main \
+	train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.hybrid_engine=False \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.partial_rollout="${partial_rollout}"
+```
+
+## Experiments
+
+### Asynchronous Training on 7B Model
+
+We used Qwen2.5-Math-7B to verify the benefits of the fully async strategy under long candidates and multiple resources.
+Using the `async stream pipeline with stale samples` strategy, we achieved about 2x performance improvement on 32 cards,
+64 cards, and 128 cards without significantly affecting experimental results.
+
+* Machine: H20
+* Model: Qwen2.5-Math-7B
+* Rollout length: max_response_length FSDP2: 28K tokens;
+* Algorithm: DAPO
+* Dataset: TRAIN_FILE: dapo-math-17k.parquet TEST_FILE: aime-2024.parquet
+* Engine: vllm+FSDP2
+* rollout.n: 16
+* ppo_mini_batch_size: 32
+* test_freq: 20
+
+* colocate sync:
+    * step: 400
+    * train_batch_size: 512
+
+* fully_async_policy
+    * total_rollout_steps: 512*400
+    * require_batches: 4
+    * trigger_parameter_sync_step: 4
+    * staleness_threshold: 0.5
+    * partial_rollout: True
+
+|  training mode   	   | resource allocation 	 | step  	  |  gen  	  | old_log_prob 	 | update_actor 	  | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 | total time<br>400 step 	 |      acc/mean@1          	      |
+|:--------------------:|:---------------------:|:--------:|:--------:|:--------------:|:---------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------:|:-------------------------------:|
+| colocate sync      	 | 32                  	 | 790.10 	 | 357.41 	 | 107.71       	 | 269.80        	 | 13h 44m                	 | 1d 3h 43m              	 | 2d 9h 22m              	 | 3d 17h 5m              	 | max: 0.3313<br>last: 0.2448  	  |
+| fully_async_policy 	 | 16:16               	 |  294.77  |  21.26   | \            	 |     313.81      |    7h 58m<br>(1.72x)     |    16h 21m<br>(1.70x)    |   1d 0h 53m<br>(2.31x)   |   1d 9h 26m<br>(2.66x)   | max: 0.3302<br>last: 0.2333   	 |
+| colocate sync      	 | 64                  	 | 365.28 	 | 150.72 	 | 70.26        	 | 133.41       	  | 10h 22m                	 | 20h 45m                	 | 1d 7h 6m               	 | 1d 17h 32m             	 | max: 0.3365<br>last:  0.2333 	  |
+| fully_async_policy 	 | 32:32               	 | 189.26 	 | 28.46  	 | \            	 | 156.98       	  | 4h 57m<br>(2.09x)      	 | 10h 14m<br>(2.03x)     	 | 16h 58m<br>(1.83x)     	 | 21h 40m<br>(1.92x)     	 | max: 0.3677<br>last: 0.3406  	  |
+| colocate sync      	 | 128                 	 | 356.30 	 | 177.85 	 | 53.92        	 | 113.81       	  | 8h 36m                 	 | 17h 56m                	 | 1d 5h 6m               	 | 1d 16h 48m             	 | max: 0.3573<br>last: 0.2958  	  |
+| fully_async_policy 	 | 64:64               	 | 150.63 	 | 33.14  	 | \            	 | 113.16       	  | 3h 13m<br>(2.67x)      	 | 6h 46m<br>(2.65x)      	 | 10h 53m<br>(2.67x)     	 | 17h 22m<br>(2.35x)     	 | max: 0.3521<br>last: 0.3094  	  |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-colocate_async?nw=nwuserhouzg
+
+### 128-card 7B Asynchronous Mode Experiment
+
+We used Qwen2.5-Math-7B to verify the effects of various modes supported by fully async.
+We can see that the benefit brought by streaming is approximately 1.6x, and after combining staleness and
+partial_rollout, the benefit reaches 2.35x.
+
+|                             mode                                         	                              | step  	  |  gen  	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 | total time<br>400 step 	 |      acc/mean@1         	      |
+|:-------------------------------------------------------------------------------------------------------:|:--------:|:--------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------------:|
+|                                          colocate sync      	                                           | 356.30 	 | 177.85 	 | 53.92        	 | 113.81       	 | 8h 36m                 	 | 17h 56m                	 | 1d 5h 6m               	 | 1d 16h 48m             	 | max: 0.3573<br>last: 0.2958  	 |
+| `stream off policy pipeline`<br>(+fully async: trigger_parameter_sync_step= 4,<br>require_batches= 4) 	 | 231.34 	 | 128.47 	 | \            	 | 98.77        	 | 4h 25m                 	 | 9h 41m                 	 | 15h 2m                 	 | 1d 1h 53m              	 | max: 0.2844<br>last: 0.2604 	  |
+|          `async stream pipeline with stale samples`<br>(+staleness_threshold=0.5)            	          |    	     |    	     |       	        |       	        |            	             |            	             |            	             |            	             |               	                |
+|        `async stream pipeline with partial rollout`<br>(+partial_rollout=True)                 	        | 150.63 	 | 33.14  	 | \            	 | 113.16       	 | 3h 13m                 	 | 6h 46m                 	 | 10h 53m                	 | 17h 22m                	 | max: 0.3521<br>last: 0.3094 	  |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-stream_stale_partial?nw=nwuserhouzg
+
+### 128-card Stale Ablation Experiment
+
+Under the `async stream pipeline with partial rollout` mode, we verified the impact of staleness settings on training
+efficiency.
+We found that the larger the staleness, the more obvious the final gains.
+We also noticed that the times for staleness values of 0.3 and 0.5 are quite close, because as the training steps
+increase, the response length changes significantly, causing training instability.
+Further analysis and optimization are needed for this issue.
+
+| staleness_threshold 	 | step  	  |  gen  	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 | total time<br>400 step 	 |     acc/mean@1         	      |
+|:---------------------:|:--------:|:--------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------:|:-----------------------------:|
+| 0                   	 | 231.34 	 | 128.47 	 | \            	 | 98.77        	 | 4h 25m                 	 | 9h 41m                 	 | 15h 2m                 	 | 1d 1h 53m              	 | max: 0.2844<br>last: 0.2604 	 |
+| 0.1                 	 | 171.30 	 | 58.17  	 | \            	 | 109.12       	 | 3h 53m                 	 | 8h 37m                 	 | 14h 25m                	 | 19h 59m                	 | max: 0.3542<br>last: 0.2979 	 |
+| 0.3                 	 | 146.11 	 | 38.88  	 | \            	 | 103.22       	 | 3h 18m                 	 | 6h 49m                 	 | 11h 40m                	 | 17h 20m                	 | max: 0.3469<br>last: 0.2865 	 |
+| 0.5                 	 | 150.63 	 | 33.14  	 | \            	 | 113.16       	 | 3h 13m                 	 | 6h 46m                 	 | 10h 53m                	 | 17h 22m                	 | max: 0.3521<br>last: 0.3094 	 |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-stream_stale_partial?nw=nwuserhouzg
+
+### 128-card 7B require_batches Ablation Experiment
+
+In multiple tests, we found that the number of samples issued each time in streaming affects the response length during
+training, which in turn affects training time. We verified the impact on results by modifying
+`async_training.require_batches`.
+
+| require_batches 	 | step  	  | gen  	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 |     acc/mean@1         	      |
+|:-----------------:|:--------:|:-------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:------------------------:|:-----------------------------:|
+| 1               	 | 203.47 	 | 30.88 	 | \            	 | 181.08       	 | 3h 31m                 	 | 8h 29m                 	 | 17h 36m                	 | max: 0.349<br>last: 0.326   	 |
+| 2               	 | 158.72 	 | 26.32 	 | \            	 | 128.08       	 | 3h 35m                 	 | 7h 38m                 	 | 13h 57m                	 | max: 0.351<br>last: 0.3406  	 |
+| 4               	 | 124.64 	 | 25.62 	 | \            	 | 95.06        	 | 3h 13m                 	 | 6h 46m                 	 | 10h 53m                	 | max: 0.3521<br>last: 0.3521 	 |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-ablation_require_batches?nw=nwuserhouzg
+
+### 30B Model Mode Experiment
+
+We achieved a 1.7x performance improvement with `async stream pipeline with staleness samples` strategy on the
+Qwen3-30B-A3B-Base model compared to the colocate setup. It is worth noting that this is far from the upper limit of
+performance gains achievable through asynchrony. Firstly, the comparative experiments used a maximum response length of
+only 8k, which is much shorter than the 20k sequence length in previous experiments, resulting in a less pronounced
+rollout tail effect. Secondly, we adopted a highly skewed resource allocation, with rollout using 96 GPUs and trainer
+using 32 GPUs, which is not an optimal configuration. During the experiments, we observed that the current verl
+implementation imposes certain constraints, such as requiring data to be evenly divisible by the number of GPUs, making
+resource adjustment less flexible. Additionally, as asynchronous training and deployment accelerate, the performance gap
+is gradually narrowing. Therefore, enabling more flexible resource allocation and dynamic resource adjustment in the
+future will be our next focus.
+
+* Machine: H20
+* Model: Qwen3-30B-A3B-Base
+* Rollout length: max_response_length : 8K tokens;
+* Algorithm: GRPO
+* Dataset: TRAIN_FILE: dapo-math-17k.parquet TEST_FILE: aime-2024.parquet
+* Engine: vllm+Megatron
+* rollout.n: 16
+* ppo_mini_batch_size: 128
+* test_freq: 20
+
+* colocate sync:
+    * step:400
+    * train_batch_size: 512
+
+* fully_async_policy
+    * total_rollout_steps: 512*400
+    * trigger_parameter_sync_step: 512/128 = 4
+    * staleness_threshold: 0.5
+    * partial_rollout: True
+
+| Training Mode      | Resource Allocation | Step   | Gen    | Old Log Prob | Ref   | Update Actor | Total Time 100 Step | Total Time 200 Step | Total Time 300 Step | Total Time 400 Step | Acc/Mean@1                  |
+|--------------------|---------------------|--------|--------|--------------|-------|--------------|---------------------|---------------------|---------------------|---------------------|-----------------------------|
+| Colocate Sync      | 128                 | 497.89 | 348.05 | 28.73        | 20.86 | 86.27        | 13h 36m             | 1d 3h 48m           | 1d 19h 4m           | 2d 11h 39m          | max: 0.3500<br>last: 0.3208 |
+| Fully Async Policy | 96:32               | 282.75 | 22.06  | \            | 50.05 | 206.63       | 6h 45m (2.01x)      | 14h 48m (1.88x)     | 1d 0h 9m (1.78x)    | 1d 10h 41m (1.72x)  | max: 0.3813<br>last: 0.3448 |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-30B?nw=nwuserhouzg           | | |
+
+### checkpoint-engine Ablation Experiment
+We tested the single-step parameter synchronization time of the checkpoint-engine on three models: Qwen2.5-Math-7B, Qwen3-30B-A3B, and Qwen3-235B-A22B, using default checkpoint-engine configurations. All experiments were performed on H20 machines, and the Megatron engine was used for training.
+| model |  trainer rank 	  | rollout rank	  | checkpoint-engine 	 | total sync time 	 |
+|:-----------------:|:--------:|:-------:|:--------------:|:--------------:|
+| Qwen2.5-Math-7B   | 4        | 4       | False      | 0.12s      |
+| Qwen2.5-Math-7B   | 4        | 4       | True      | 0.02s      |
+|  Qwen3-30B-A3B     | 16        | 16       | False      | 15.76s   |
+|  Qwen3-30B-A3B     | 16        | 16       | True      | 4.38s   |
+|  Qwen3-235B-A22B    | 64        | 64       | False      | 58.57s   |
+|  Qwen3-235B-A22B    | 64        | 64       | True      | 23.70s   |
+
+
+### use_trainer_do_validate Experiment
+We tested the effect of setting `use_trainer_do_validate=True` on the training process. The results show that setting
+this parameter to True can reduce the validation time overhead and trainer node idle time.
+We used Qwen2.5-Math-7B to verify the benefits of `use_trainer_do_validate=True` on the training process, we achieved about 2x performance improvement on validation time, and the trainer node idle time is reduced by about 40%.
+
+* Machine: H20
+* Model: Qwen2.5-Math-7B
+* Rollout length: max_response_length FSDP2: 10K tokens;
+* Algorithm: DAPO
+* Dataset: TRAIN_FILE: dapo-math-17k.parquet TEST_FILE: aime-2024.parquet
+* Engine: vllm+FSDP2
+* rollout.n: 16
+* ppo_mini_batch_size: 32
+* test_freq: 10
+
+* fully_async_policy
+    * total_rollout_steps: 512*400
+    * require_batches: 4
+    * trigger_parameter_sync_step: 4
+    * staleness_threshold: 0.5
+    * partial_rollout: True
+
+|  training mode  | resource allocation | step  |  gen  | old_log_prob | update_actor | validate time | total time<br>50 step | acc/mean@2 |
+|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|
+| colocate sync      | 16  |  484.623  |  52.939	 |   0	 |   430.263   |  205.080  	 |     7h9m  	 |     22.6     |
+| fully_async_policy | 8:8 |  489.953  |  52.622	 |   0	 |   435.874   |  95.699  	 |     7h2m  	 |     21.0    |
+
+
+## Multi-Turn Tool Calling
+
+Referencing **recipe/retool** and **ToolAgentLoop**, we implemented **AsyncPartialToolAgentLoop**, a multi-turn
+tool-calling loop that supports partial_rollout for **fully_async_policy**.
+
+### Core Design
+
+`AsyncPartialToolAgentLoop` inherits from `ToolAgentLoop` and is adapted for the asynchronous training mode of
+`fully_async_policy`. When `partial_rollout=True`, the Rollouter interrupts ongoing generation tasks before
+synchronizing parameters with the Trainer. `AsyncPartialToolAgentLoop` is capable of:
+
+1. **Interrupting Tasks**: Responding to an interrupt signal to save the current state. Currently, interruptions occur
+   during the `GENERATING` process or after other states have completed.
+2. **Resuming Tasks**: Resuming execution from the saved state after parameter synchronization is complete, rather than
+   starting over.
+
+### How to Use
+
+RL training with multi-turn tool calling in `fully_async_policy` is similar to `recipe/retool`. It is enabled by
+specifying `multi_turn` configurations in the config file.
+
+1. **SFT Stage**: First, the model should undergo SFT to learn how to follow tool-calling format instructions.
+2. **Multi-turn Configuration**: In the `fully_async_policy` training configuration, set the following parameters:
+   ```yaml
+   actor_rollout_ref:
+     rollout:
+       multi_turn:
+         enable: True # AsyncPartialToolAgentLoop will be used by default in fully_async_policy mode
+         # Other multi_turn related configurations
+   ```
+3. **Async Parameters**: To improve efficiency, enable `partial_rollout` and `staleness_threshold` when using multi-turn
+   tool calling:
+   ```yaml
+   async_training:
+     partial_rollout: True
+     staleness_threshold: 0.5
+     # Other async parameters
+   ```
+4. **Example**: See `recipe/fully_async_policy/shell/dapo_7b_async_retool.sh`.
+
+### Experimental Results
+
+To validate the performance of `fully_async_policy` on multi-turn tool-calling tasks, we compared it with the standard
+`colocate` synchronous mode. Key parameter settings are as follows.
+
+* **SFT Model**: Based on `Qwen2.5-7B-Instruct`, trained for 6 epochs on the `ReTool-SFT` dataset
+* **RL Algorithm**: DAPO
+* **Dataset**:
+    * Train: `DAPO-Math-17k`
+    * Test: `aime_2025`
+* **Resource and Mode Comparison**:
+    * `colocate sync`: 32 H20 gpus
+    * `fully_async_policy`: 16 gpus for Trainer + 16 gpus for Rollouter
+* **Key Configurations**:
+    1. **Tool Calling Configuration**:
+        * `multi_turn.enable: True`
+        * `multi_turn.max_user_turns: 16`
+        * `multi_turn.max_assistant_turns: 16`
+        * `multi_turn.tool_config_path: recipe/retool/sandbox_fusion_tool_config.yaml`
+    2. **`colocate sync` Configuration**:
+        * `ppo_mini_batch_size: 16`
+        * `train_batch_size: 64`
+    3. **`fully_async_policy` Configuration**:
+        * `ppo_mini_batch_size: 16`
+        * `trigger_parameter_sync_step: 4`
+        * `require_batches: 1`
+        * `staleness_threshold: 1`
+        * `partial_rollout: True`
+
+|  training mode   	   | Resource allocation 	 |  step  	  |  gen   	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 |   aime_2025<br>acc/mean@30  	   |
+|:--------------------:|:---------------------:|:---------:|:---------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:-------------------------------:|
+| colocate           	 | 32                  	 | 375.47  	 | 228.03 	  | 35.19        	 | 111.84       	 | 9h 46m                 	 | 22h 28m                	 | start:0.1078<br>last:0.2056   	 |
+| fully_async_policy 	 | 16: 16              	 | 221.36 	  | 40.59   	 | \            	 | 179.58       	 | 6h 19m<br>(1.55x)      	 | 14h 4m<br>(1.60x)      	 |   start:0.11<br>last:0.2044 	   |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-multiturn-tool?nw=nwuserhouzg
+
+## Future Plans
+
+* GRPO experiments
+* Megatron adaptation
+* SGLang integration
+* Transfer queue integration
+* Asynchronous parameter synchronization
+* AReaL asynchronous algorithm implementation
+* TPPO algorithm implementation
+* Multi-turn and Tool support

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/README_zh.md

@@ -0,0 +1,517 @@
+# Recipe: Fully Async Policy Trainer
+
+**Author:**  `https://github.com/meituan-search`
+
+Last updated: 12/15/2025.
+
+本文档介绍了完全异步PPO训练系统，该系统实现了 Trainer 和 Rollouter 的完全解耦，支持异步样本生成和训练。
+在该系统下，我们使用128卡训练qwen2.5-7B模型取得了2.35x-2.67x的性能提升,同时效果没有显著受到影响。
+
+## Introduction
+
+### Background
+
+rollout和train分离架构相较于colocate的架构能够更加灵活地分配资源，设计更加灵活的训练逻辑，从而处理长尾等问题带来的GPU利用率低，训练效率低的问题。
+one_step_off_policy通过分离架构的设计并进行rollout和train一轮异步的训练方法，缓解了rollout时间过长的问题，并在训练效率上取得了一些收益，
+但其强制使用一轮异步的数据，存在不够灵活等问题，而且并不能完全去除长尾对训练效率带来的的影响；在其他框架如areal、Magistral、streamrl、asyncflow上，
+已经基于分离架构实现了异步训练、流式训练，并取得了收益；我们借鉴其方法，在verl上进行了实现。fully_async_policy支持异步、流式、partial
+rollout的训练， 通过合理设置资源分配情况、参数同步频率等参数，fully_async_policy能够显著提高训练效率。
+
+> Magistral https://arxiv.org/abs/2506.10910
+>
+> AReaL: A Large-Scale Asynchronous Reinforcement Learning System for Language
+> Reasoning https://arxiv.org/abs/2505.24298
+>
+> StreamRL: Scalable, Heterogeneous, and Elastic RL for LLMs with Disaggregated Stream
+> Generation https://arxiv.org/abs/2504.15930
+>
+> AsyncFlow: An Asynchronous Streaming RL Framework for Efficient LLM Post-Training https://arxiv.org/abs/2507.01663
+>
+
+### 核心贡献
+
+* **资源隔离**：与使用hybrid_engine不同，Rollouter和Trainer使用分离的计算资源，需要分别指定所占用的资源。
+* **生成与训练并行**：Trainer在训练的同时，Rollouter在生成新的样本。
+* **多步异步**: 相比 one step off policy 支持0.x步到多步的异步设定，异步方案更加灵活。
+* **nccl参数同步**：基于nccl通信原语，参考[checkpoint-engine](https://github.com/MoonshotAI/checkpoint-engine)实现Rollouter与Trainer间的高效参数同步。
+* **Stream推理与训练**：Rollouter逐样本生成数据，同时数据传输以单个sample为最小传输单位。
+* **异步训练与新鲜度控制**：通过设置参数async_training.staleness_threshold，支持使用旧参数生成的样本进行训练。
+* **PartialRollout**: Rollouter推理过程支持partial rollout逻辑，通过参数同步时，添加`sleep()`和`resume()`
+  逻辑，保存进行中的rollout的样本，并在下一次rollout中继续使用，减少参数同步等待进行中的任务结束时间。
+
+目前支持使用模式为 megatron/fsdp+vllm。vllm必须使用基于AgentLoop的server模式。
+
+## 设计
+
+fully_async_policy的整体架构如下图所示，fully_async_policy主要由Rollouter、MessageQueue、Trainer、ParameterSynchronizer四部分组成。
+
+![fully_async_policy_structure](
+https://github.com/ArronHZG/verl-community/blob/main/docs/fully_async_policy_structure.svg?raw=true)
+
+1. Rollouter逐样本生成序列，并将生成的sample放入MessageQueue中，生产的速度受新鲜度控制。
+2. MessageQueue用于暂存Rollouter生成的sample。
+3. Trainer逐样本从MessageQueue中获取，获取到`require_batches*ppo_mini_batch_size`
+   数量的样本后，就会进行训练，训练async_training.trigger_parameter_sync_step轮后，触发与Rollouter的一次参数同步。
+4. ParameterSynchronizer 实现了Nccl的同步参数同步能力。
+
+当前方案对比base的收益来源，在于colocate情况下，rollout使用更多的资源无法解决长尾样本带来的空闲，
+当我们进行资源隔离后，rollout的时间和train的时间都可能相较于之前更长（因为使用的资源变少了），
+但是相互之间的耗时overlap，端到端的耗时反而有所缩减。
+
+![fully_async_policy_revenue](
+https://github.com/ArronHZG/verl-community/blob/main/docs/fully_async_policy_revenue.svg?raw=true)
+
+## 使用方式
+
+### 参数说明
+
+| super params                                         | implication                                                     |
+|------------------------------------------------------|-----------------------------------------------------------------|
+| `trainer.nnodes`                                     | Trainer的node数量                                                  |
+| `trainer.n_gpus_per_node`                            | Trainer每个node上gpu的数量                                            |
+| `rollout.nnodes`                                     | Rollouter的node数量                                                |
+| `rollout.n_gpus_per_node`                            | Rollouter每个node上gpu的数量                                          |
+| `data.train_batch_size`                              | 在fully async策略中，该值不生效（默认设置为0）                                   |
+| `data.gen_batch_size`                                | 在fully async策略中，使用流式的样本生产逻辑（默认设置为1)                             |
+| `rollout.total_rollout_steps`                        | 总的rollout的sample数量                                              |
+| `rollout.test_freq`                                  | Rollouter每更新多少次参数，进行一次validation                                |
+| `actor_rollout_ref.actor.ppo_mini_batch_size`        | The ppo_mini_batch_size is a global num across all workers/gpus |
+| `async_training.require_batches`                     | FullyAsyncTrainer一次性获取的ppo_mini_batch_size的数量                   |
+| `async_training.trigger_parameter_sync_step`         | 表示FullyAsyncTrainer进行多少次本地更新后,进行一次参数同步                          |
+| `async_training.staleness_threshold`                 | 新鲜度控制                                                           |
+| `async_training.partial_rollout`                     | 是否进行partial_rollout                                             |
+| `async_training.use_rollout_log_probs`               | 使用rollout产生的log_probs                                           |
+| `async_training.compute_prox_log_prob`（experimental） | 是否在train阶段，使用train模型的参数计算token的 log_prob                        |
+| `async_training.checkpoint_engine.enable`| 是否开启checkpoint_engine模式的加速，默认值True |
+| `async_training.checkpoint_engine.overlap_broadcast_and_consume` | 启动checkpoint_engine时，是否在参数同步时在broadcast和加载之间使用流水，默认值False|
+| `async_training.checkpoint_engine.device_buffer_size_M` | 启动checkpoint_engine时，组装的bucket的大小(MB)，默认为4096 |
+| `async_training.use_trainer_do_validate` | 是否使用Trainer的do_validate方法进行validation，默认值False |
+
+**进一步的解释：**
+
+* `rollout.total_rollout_steps`
+
+  与 colocate 相比，数量可以通过 train_batch_size 与 step 相乘对齐:
+  `rollout.total_rollout_steps = data.train_batch_size * step`。
+
+* `async_training.trigger_parameter_sync_step`
+
+  在fully async策略中，表示Trainer进行多少次本地更新后（也就是获取多少次`require_batches * ppo_mini_batch_size`数量样本），
+  与Rollouter之间进行一次参数同步。
+  每两次Rollouter和Trainer参数同步之间，Trainer将会处理`trigger_parameter_sync_step* require_batches\
+  ppo_mini_batch_size`份sample。
+  如果为了与colocate在公平的情况下对比速度，trigger_parameter_sync_step应该设置为 `data.train_batch_size / (
+  require_batches * ppo_mini_batch_size)`。
+
+* `async_training.staleness_threshold`
+
+  在fully async策略中，表示最大允许使用的staleness样本的比例。
+
+    * staleness_threshold=0，表示同步训练。
+      Rollouter两次参数更新之间将会生成固定数量的样本，样本数为：
+      $$rollout\_num = (trigger\_parameter\_sync\_step*require\_batches*ppo\_mini\_batch\_size)$$
+    * staleness_threshold>0，表示异步训练， 可以设置为小数，支持更灵活的异步调用。
+      Rollouter两次参数更新之间将会最多生成的样本数为：
+      $$rollout\_num = (1+staleness\_threshold)*(trigger\_parameter\_sync\_step*require\_batches*ppo\_mini\_batch\_size) - num\_staleness\_sample $$
+
+  num_staleness_sample 表示上一次rollout多生成的陈旧样本数。
+
+  由于是流式系统，rollout持续生成，trainer持续消费。如果rollouter较慢，trainer会更早触发参数同步，rollouter并不会实际生产rollout_num个样本。
+  当rollout 足够快时，staleness_threshold设置为1，基本上等价于one_step_off policy。
+  为了避免过期样本太多影响训练精度，建议该值设置小于1。
+
+* `async_training.partial_rollout`
+
+  partial_rollout只会在staleness_threshold>0时才实际上起作用。
+
+* `async_training.use_rollout_log_probs`
+
+  在强化学习算法中，log_probs与参数版本，token都存在隐性的相关性。由于PPO/GRPO/DAPO等算法的设定，我们在计算重要性采样时，
+  即 old_log_prob必须使用rollout参数及token所对应log_probs，才能保证算法的正确性。在fully
+  async策略中，我们默认old_log_prob是有rollout所计算的，而不是由trainer所计算。
+
+* `async_training.require_batches`
+
+  在流式训练中，require_batches 应该设置为1，表示生产够ppo_mini_batch_size样本后，就进行训练。
+  在实际测试中，我们发现，如果单次下发的样本较少，由于数据分发的顺序，会导致训练不稳定，response 长度变长。
+  在这里，我们额外提供 require_batches 进行流式分发，单次参与训练的样本数量控制。
+
+* `async_training.compute_prox_log_prob` （experimental）
+
+  我们在训练过程中，观测到随着训练的进行，训练后期指标和response长度可能会出现不稳定的情况，
+  这里我们可以使用 [Rollout Importance Sampling](https://verl.readthedocs.io/en/latest/advance/rollout_is.html) 的技术进行
+  重要性采样，缓解这一问题。为了使用 `Rollout Importance Sampling` 我们需要使用训练引擎使用当前的参数版本计算old_log_prob，此开关需要打开。
+  此外，在 mode d (async stream pipeline with partial rollout) 的情况下开启 `compute_prox_log_prob` 以及
+  `Rollout Importance Sampling` 后，我们的实现已近似Areal的 `Decoupled PPO`。
+
+* `async_training.checkpoint_engine.enable`
+  
+  开启checkpoint engine后，相较于原始的逐tensor的参数同步方式，同步时间开销普遍可以降低60%以上。但是组装bucket会带来额外的临时显存开销。
+
+* `async_training.checkpoint_engine.overlap_broadcast_and_consume`
+
+  开启参数broadcast和load_weights之间的流水后，会进一步额外申请更多显存。由于目前分析参数同步的主要耗时并非来自broadcast和load_weights阶段，而是在参数生成阶段（由megatron或FSDP），因此该开关默认关闭。
+
+* `async_training.checkpoint_engine.device_buffer_size_M`
+  
+  控制开启checkpoint engine后，用于同步的显存buffer大小。实际的`bucket_size` = `max(device_buffer_size_M, 最大参数tensor size)`
+  * 在开启`overlap_broadcast_and_consume`时，trainer节点的临时额外显存开销为 `3 * bucket_size`, rollout节点的临时额外显存开销为`2 * bucket_size`。
+  * 在关闭`overlap_broadcast_and_consume`时，trainer节点的临时额外显存开销为 `2 * bucket_size`, rollout节点的临时额外显存开销为`1 * bucket_size`。
+
+* `async_training.use_trainer_do_validate`
+
+  控制是否使用trainer的`do_validate`方法进行validation。
+  如果设置为True，trainer会在每次参数更新后，调用`do_validate`方法进行validation。
+  如果设置为False，trainer不会调用`do_validate`方法。
+
+### 模式支持
+
+1. on policy pipeline:
+    1. **trigger_parameter_sync_step=1，staleness_threshold=0**
+    2. Rollouter一次生产`require_batches*ppo_mini_batch_size`
+       的samples，Trainer获取这些samples后进行训练，训练完后Trainer和Rollouter之间进行一次参数同步;
+    3. 在rollout阶段，如果存在长尾的样本，但是rollout样本数较少时，较短的样本无法填充到空闲的资源中，会造成一定的资源浪费。
+    4. 如图a所示；
+
+2. stream off policy pipeline:
+    1. **trigger_parameter_sync_step>1，staleness_threshold=0**
+    2. 将会进行同步的流式训练，Rollouter一次生产`require_batches*ppo_mini_batch_size*trigger_parameter_sync_step`
+       的samples，Trainer每获取`require_batches*ppo_mini_batch_size`
+       就进行一次本地训练，训练trigger_parameter_sync_step次后，Trainer和Rollouter之间进行一次参数同步;
+    3. 相较于a，由于一次生成的样本更多，资源的空闲会更低。
+    4. 在一次step训练中，会存在两次资源闲置的时间，分别是在第一次获取样本时，train等待`require_batches*ppo_mini_batch_size`
+       个样本生产，以及最后一次参数更新时，rollout等待训练完成。
+    5. 如图b所示；
+
+3. async stream pipeline with staleness samples:
+    1. **trigger_parameter_sync_step>=1，staleness_threshold>0，partial_rollout=Flase**
+    2. Rollouter在每次参数更新后将计划最多生产rollout_num个样本（实际根据rollout速度，生成的样本可能会少与这个值）。
+    3. 如果rollout过程比较快，Rollouter将会在参数同步前额外生成一部分样本num_stale_samples，用于参数同步后立即给Trainer使用。
+       触发参数同步时，如果Rollouter有正在生产的任务，将会等待任务完成，同时不会添加新的任务；
+    4. 相较于b，除第一次step训练外，后续的训练都不会有wait first batch rollout finish的时间，但是会有wait active task
+       finish的时间。
+    5. 如图c所示；
+
+4. async stream pipeline with partial rollout:
+    1. **trigger_parameter_sync_step>=1，staleness_threshold>0，partial_rollout=True**
+    2. 相较于c，触发参数同步时，Rollouter如果有正在生产的sample，会打断rollout过程并进行参数同步，被中断的sample会在参数同步后继续生成。减少了wait
+       active task finish的时间。
+    3. 如图d所示；
+
+![fully_async_policy_mode](
+https://github.com/ArronHZG/verl-community/blob/main/docs/fully_async_policy_mode.svg?raw=true)
+
+### 关键指标
+
+| metrics                                        | implication                                               |
+|------------------------------------------------|-----------------------------------------------------------|
+| `trainer/idle_ratio`                           | Trainer闲置率                                                |
+| `rollouter/idle_ratio`                         | Rollouter闲置率                                              |
+| `fully_async/count/stale_samples_processed`    | 训练使用的旧sample总数                                            |
+| `fully_async/count/stale_trajectory_processed` | 训练使用的旧trajectory总数(一个sample会生产rollout.n条trajectory)       |
+| `fully_async/partial/total_partial_num`        | 两次trigger_parameter_sync_step之间Trainer处理的partial样本数       |
+| `fully_async/partial/partial_ratio`            | 两次trigger_parameter_sync_step之间Trainer处理的partial样本的比例     |
+| `fully_async/partial/max_partial_span`         | 两次trigger_parameter_sync_step之间Trainer处理的partial样本的最大参数跨度 |
+
+### 调参建议
+
+* 资源分配与调整:
+    * 合理的资源分配是获得好的训练效率的前提。理想的资源分配情况应该是使得Rollout的时间和Train的时间接近，从而使得整个训练过程流水气泡最小，
+      避免资源闲置，同时Trainer不会使用旧样本。在真实训练场景下，可以根据实际训练过程中rollout和train的空闲时间调整资源分配，
+      可从rollouter/idle_ratio和trainer/idle_ratio获得，如果rollouter/idle_ratio较高trainer/idle_ratio较低，
+      应该增多Trainer的资源减少Rollouter的资源，反之亦然。
+
+* 关键参数：
+    * staleness_threshold: 设置太大会导致较多的旧样本使用，影响模型效果，建议设置小于1。
+    * require_batches：越接近1，越接近纯流式过程，训练过程中bubble越小，能够在速度上获得更快的加速效果，但会对样本的处理顺序产生影响；
+    * trigger_parameter_sync_step: 设置的越小越接近on policy，但会导致频繁的参数同步，长尾样本浪费的资源无法被短样本填充，资源利用率低。
+      设置的越大有更高的计算效率，但是精度上会受到off policy的影响。
+    * rollout.test_freq: 会占用Rollouter资源，不建议设置太小。
+
+* 模式选择：通过调整不同的参数，Fully Async架构支持不同程度上的优化加速，适用于不同场景的任务。
+    * 对于小规模任务，需要保证训练的稳定性和 on-policy 性，对速度要求不高的场景，可以尝试使用on policy pipeline的模式（模式1）。
+    * 对于需要提高训练吞吐量，但对 staleness 敏感的场景，可以尝试使用 stream off policy pipeline 的模式。即通过
+      设置trigger_parameter_sync_step>1 ，提高 训练效率，但仍保持同步机制 (staleness_threshold=0 )（模式2）。
+    * 对于大规模任务，对训练速度有较高要求，且可以容忍一定 off-policy 程度、staleness的场景，可以设置staleness_threshold>
+      0、partial_rollout=True提高训练效率，使用 async stream pipeline 模式（模式 3 或 4）。
+
+### 快速开始
+
+```shell
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*400)))
+test_freq=10
+staleness_threshold=0
+trigger_parameter_sync_step=16
+partial_rollout=False
+
+
+python -m recipe.fully_async_policy.fully_async_main \
+	train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.hybrid_engine=False \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.partial_rollout="${partial_rollout}"
+```
+
+## 实验
+
+### 在7B模型上进行异步训练
+
+我们使用 Qwen2.5-Math-7B 验证 fully async 策略在长候选下，多种资源下的收益情况。
+使用`async stream pipeline with staleness samples` 策略，我们在32卡，64卡，128卡都取得2x左右的性能提升，同时没有显著影响实验效果。
+
+* 机器：H20
+* 模型：Qwen2.5-Math-7B
+* rollout长度：max_response_length FSDP2: 28K tokens;
+* 算法：DAPO
+* 数据集： TRAIN_FILE: dapo-math-17k.parquet TEST_FILE: aime-2024.parquet
+* engine: vllm+FSDP2
+* rollout.n: 16
+* ppo_mini_batch_size: 32
+* test_freq: 20
+
+* colocate sync:
+    * step: 400
+    * train_batch_size: 512
+
+* fully_async_policy
+    * total_rollout_steps: 512*400
+    * require_batches: 4
+    * trigger_parameter_sync_step: 4
+    * staleness_threshold: 0.5
+    * partial_rollout: True
+
+|  training mode   	   | resource allocation 	 | step  	  |  gen  	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 | total time<br>400 step 	 |      acc/mean@1          	      |
+|:--------------------:|:---------------------:|:--------:|:--------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------:|:-------------------------------:|
+| colocate sync      	 | 32                  	 | 790.10 	 | 357.41 	 | 107.71       	 | 269.80      	 | 13h 44m                	 | 1d 3h 43m              	 | 2d 9h 22m              	 | 3d 17h 5m              	 | max: 0.3313<br>last: 0.2448  	  |
+| fully_async_policy 	 | 16:16               	 |  294.77  |  21.26   | \            	 |      313.81     |    7h 58m<br>(1.72x)     |    16h 21m<br>(1.70x)    |   1d 0h 53m<br>(2.31x)   |   1d 9h 26m<br>(2.66x)   | max: 0.3302<br>last: 0.2333   	 |
+| colocate sync      	 | 64                  	 | 365.28 	 | 150.72 	 | 70.26        	 | 133.41       	 | 10h 22m                	 | 20h 45m                	 | 1d 7h 6m               	 | 1d 17h 32m             	 | max: 0.3365<br>last:  0.2333 	  |
+| fully_async_policy 	 | 32:32               	 | 189.26 	 | 28.46  	 | \            	 | 156.98       	 | 4h 57m<br>(2.09x)      	 | 10h 14m<br>(2.03x)     	 | 16h 58m<br>(1.83x)     	 | 21h 40m<br>(1.92x)     	 | max: 0.3677<br>last: 0.3406  	  |
+| colocate sync      	 | 128                 	 | 356.30 	 | 177.85 	 | 53.92        	 | 113.81       	 | 8h 36m                 	 | 17h 56m                	 | 1d 5h 6m               	 | 1d 16h 48m             	 | max: 0.3573<br>last: 0.2958  	  |
+| fully_async_policy 	 | 64:64               	 | 150.63 	 | 33.14  	 | \            	 | 113.16       	 | 3h 13m<br>(2.67x)      	 | 6h 46m<br>(2.65x)      	 | 10h 53m<br>(2.67x)     	 | 17h 22m<br>(2.35x)     	 | max: 0.3521<br>last: 0.3094  	  |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-colocate_async?nw=nwuserhouzg
+
+### 128卡 7B 异步模式实验
+
+我们使用 Qwen2.5-Math-7B 验证 fully async 所支持的各个模式的效果。
+我们可以看到 stream 带来的收益大约1.6x，叠加 staleness 和 partial_rollout 后，收益为2.35x。
+
+|                             mode                                         	                              | step  	  |  gen  	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 | total time<br>400 step 	 |      acc/mean@1         	      |
+|:-------------------------------------------------------------------------------------------------------:|:--------:|:--------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------------:|
+|                                          colocate sync      	                                           | 356.30 	 | 177.85 	 | 53.92        	 | 113.81       	 | 8h 36m                 	 | 17h 56m                	 | 1d 5h 6m               	 | 1d 16h 48m             	 | max: 0.3573<br>last: 0.2958  	 |
+| `stream off policy pipeline`<br>(+fully async: trigger_parameter_sync_step= 4,<br>require_batches= 4) 	 | 231.34 	 | 128.47 	 | \            	 | 98.77        	 | 4h 25m                 	 | 9h 41m                 	 | 15h 2m                 	 | 1d 1h 53m              	 | max: 0.2844<br>last: 0.2604 	  |
+|        `async stream pipeline with staleness samples`<br>(+staleness_threshold=0.5)            	        |    	     |    	     |       	        |       	        |            	             |            	             |            	             |            	             |               	                |
+|        `async stream pipeline with partial rollout`<br>(+partial_rollout=True)                 	        | 150.63 	 | 33.14  	 | \            	 | 113.16       	 | 3h 13m                 	 | 6h 46m                 	 | 10h 53m                	 | 17h 22m                	 | max: 0.3521<br>last: 0.3094 	  |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-stream_stale_partial?nw=nwuserhouzg
+
+### 128卡 stale 消融实验
+
+在 `async stream pipeline with partial rollout` 模式下，我们验证 staleness 的设置对于训练效率的影响。
+我们可以发现，staleness 越大，最终取得的收益越明显。
+同时我们也注意到 staleness 取 0.3 和 0.5 的时间比较接近，原因是随着训练步数的增量，response 长度变化较大，训练出现了不稳定的问题。
+后续还需要针对该问题进行进一步的分析和优化。
+
+| staleness_threshold 	 | step  	  |  gen  	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 | total time<br>400 step 	 |     acc/mean@1         	      |
+|:---------------------:|:--------:|:--------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:------------------------:|:------------------------:|:-----------------------------:|
+| 0                   	 | 231.34 	 | 128.47 	 | \            	 | 98.77        	 | 4h 25m                 	 | 9h 41m                 	 | 15h 2m                 	 | 1d 1h 53m              	 | max: 0.2844<br>last: 0.2604 	 |
+| 0.1                 	 | 171.30 	 | 58.17  	 | \            	 | 109.12       	 | 3h 53m                 	 | 8h 37m                 	 | 14h 25m                	 | 19h 59m                	 | max: 0.3542<br>last: 0.2979 	 |
+| 0.3                 	 | 146.11 	 | 38.88  	 | \            	 | 103.22       	 | 3h 18m                 	 | 6h 49m                 	 | 11h 40m                	 | 17h 20m                	 | max: 0.3469<br>last: 0.2865 	 |
+| 0.5                 	 | 150.63 	 | 33.14  	 | \            	 | 113.16       	 | 3h 13m                 	 | 6h 46m                 	 | 10h 53m                	 | 17h 22m                	 | max: 0.3521<br>last: 0.3094 	 |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-ablation_stale?nw=nwuserhouzg
+
+### 128卡 7B require_batches 消融实验
+
+在多次测试下，我们发现流式每次下发样本的数量会影响训练的response长度，进而影响训练时长，我们通过修改
+`async_training.require_batches` 验证对与结果的影响。
+
+| require_batches 	 | step  	  | gen  	  | old_log_prob 	 | update_actor 	 | total time<br>100 step 	 | total time<br>200 step 	 | total time<br>300 step 	 |     acc/mean@1         	      |
+|:-----------------:|:--------:|:-------:|:--------------:|:--------------:|:------------------------:|:------------------------:|:------------------------:|:-----------------------------:|
+| 1               	 | 203.47 	 | 30.88 	 | \            	 | 181.08       	 | 3h 31m                 	 | 8h 29m                 	 | 17h 36m                	 | max: 0.349<br>last: 0.326   	 |
+| 2               	 | 158.72 	 | 26.32 	 | \            	 | 128.08       	 | 3h 35m                 	 | 7h 38m                 	 | 13h 57m                	 | max: 0.351<br>last: 0.3406  	 |
+| 4               	 | 124.64 	 | 25.62 	 | \            	 | 95.06        	 | 3h 13m                 	 | 6h 46m                 	 | 10h 53m                	 | max: 0.3521<br>last: 0.3521 	 |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-ablation_require_batches?nw=nwuserhouzg
+
+### 30B模型模式实验
+
+我们在 Qwen3-30B-A3B-Base 模型上通过`async stream pipeline with staleness samples` 策略，相比于 colocate 方案取得了 1.7
+倍的性能提升。值得说明的是，这距离异步方式所能带来的性能提升上限还有很大空间。首先，对比实验中使用的最大响应长度仅为
+8k，这远低于此前实验的 20k 序列长度，因此 rollout 的长尾效应并不明显。其次，我们采用了极为倾斜的资源分配方案，rollout 使用了
+96 张 GPU，而 trainer 仅使用了 32 张 GPU，这并不是最优的配置。在实验过程中，我们观察到当前的 verl 实现存在一些限制，比如要求数据必须能被
+GPU 数量整除，这使得资源调整的灵活性受到影响。此外，随着异步训练和部署的加速，性能差距也在逐渐缩小。因此，未来我们将重点关注如何实现更灵活的资源分配和动态调整资源。
+
+* 机器：H20
+* 模型：Qwen3-30B-A3B-Base
+* rollout长度：max_response_length : 8K tokens;
+* 算法： GRPO
+* 数据集： TRAIN_FILE: dapo-math-17k.parquet TEST_FILE: aime-2024.parquet
+* Engine: vllm+Megatron
+* rollout.n: 16
+* ppo_mini_batch_size: 128
+* test_freq: 20
+
+* colocate sync:
+    * step:400
+    * train_batch_size: 512
+
+* fully_async_policy
+    * total_rollout_steps: 512*400
+    * trigger_parameter_sync_step: 512/128 = 4
+    * staleness_threshold: 0.5
+    * partial_rollout: True
+
+| Training Mode        | Resource Allocation | Step    | Gen    | Old Log Prob | Ref    | Update Actor | Total Time 100 Step | Total Time 200 Step | Total Time 300 Step | Total Time 400 Step | Acc/Mean@1                 |
+|----------------------|--------------------|---------|--------|--------------|--------|--------------|---------------------|---------------------|---------------------|---------------------|-----------------------------|
+| Colocate Sync        | 128                | 497.89  | 348.05 | 28.73        | 20.86  | 86.27        | 13h 36m             | 1d 3h 48m           | 1d 19h 4m           | 2d 11h 39m          | max: 0.3500<br>last: 0.3208 |
+| Fully Async Policy   | 96:32              | 282.75  | 22.06  | \            | 50.05  | 206.63       | 6h 45m (2.01x)      | 14h 48m (1.88x)     | 1d 0h 9m (1.78x)    | 1d 10h 41m (1.72x)  | max: 0.3813<br>last: 0.3448 |
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-30B?nw=nwuserhouzg
+
+### checkpoint-engine参数同步消融实验
+我们在Qwen2.5-Math-7B，Qwen3-30B-A3B和Qwen3-235B-A22B三个模型上测试了checkpoint-engine参数同步的单步参数同步耗时，使用的参数均为默认参数配置。实验均在H20机器上完成，并使用megatron训练引擎。
+| model |  trainer rank 	  | rollout rank	  | checkpoint-engine 	 | total sync time 	 |
+|:-----------------:|:--------:|:-------:|:--------------:|:--------------:|
+| Qwen2.5-Math-7B   | 4        | 4       | False      | 0.12s      |
+| Qwen2.5-Math-7B   | 4        | 4       | True      | 0.02s      |
+|  Qwen3-30B-A3B     | 16        | 16       | False      | 15.76s   |
+|  Qwen3-30B-A3B     | 16        | 16       | True      | 4.38s   |
+|  Qwen3-235B-A22B    | 64        | 64       | False      | 58.57s   |
+|  Qwen3-235B-A22B    | 64        | 64       | True      | 23.70s   |
+
+### use_trainer_do_validate 实验测试
+我们在Qwen2.5-Math-7B模型上测试了`use_trainer_do_validate`参数的影响。这个结果展示使用`use_trainer_do_validate=True`可以减少验证时间开销，并且训练器节点的空闲时间也减少了。
+
+* Machine: H20
+* Model: Qwen2.5-Math-7B
+* Rollout length: max_response_length FSDP2: 10K tokens;
+* Algorithm: DAPO
+* Dataset: TRAIN_FILE: dapo-math-17k.parquet TEST_FILE: aime-2024.parquet
+* Engine: vllm+FSDP2
+* rollout.n: 16
+* ppo_mini_batch_size: 32
+* test_freq: 10
+
+* fully_async_policy
+    * total_rollout_steps: 512*400
+    * require_batches: 4
+    * trigger_parameter_sync_step: 4
+    * staleness_threshold: 0.5
+    * partial_rollout: True
+
+|  training mode  | resource allocation | step  |  gen  | old_log_prob | update_actor | validate time | total time<br>50 step | acc/mean@2 |
+|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|:---------------:|
+| colocate sync      | 16  |  484.623  |  52.939	 |   0	 |   430.263   |  205.080  	 |     7h9m  	 |     22.6     |
+| fully_async_policy | 8:8 |  489.953  |  52.622	 |   0	 |   435.874   |  95.699  	 |     7h2m  	 |     21.0    |
+
+
+## 多轮工具调用
+
+参考 **recipe/retool** 和 **ToolAgentLoop**，我们为 **fully_async_policy** 实现了支持partial rollout的多轮工具调用循环 *
+*AsyncPartialToolAgentLoop**。
+
+### 核心设计
+
+`AsyncPartialToolAgentLoop` 继承自 `ToolAgentLoop`，其核心是适配了 `fully_async_policy` 的异步训练模式。当
+`partial_rollout=True` 时，Rollouter 在与 Trainer 同步参数前会中断正在进行的生成任务。`AsyncPartialToolAgentLoop` 能够：
+
+1. **中断任务**: 响应中断信号，保存当前的生成状态。目前，中断会发生在GENERATING过程中，或其他状态结束后；
+2. **恢复任务**: 在参数同步完成后，从保存的状态恢复，继续执行，而不是从头开始。
+
+### 使用方法
+
+`fully_async_policy`多轮与工具调用的RL训练与 `recipe/retool` 类似，通过在配置文件中指定 `multi_turn` 相关配置来启用。
+
+1. **SFT 阶段**: 首先，需要对模型进行 SFT训练，使其具备遵循工具调用格式指令的能力。
+2. **配置启用**: 在 `fully_async_policy` 的训练配置中，设置以下参数:
+   ```yaml
+   actor_rollout_ref:
+     rollout:
+       multi_turn:
+         enable: True # 在fully_async_policy模式下将默认使用AsyncPartialToolAgentLoop
+         # 其他 multi_turn 相关配置
+   ```
+3. **配置async参数**: 为提高效率，在启用多轮工具调用时，同时开启 `partial_rollout`和`staleness_threshold`：
+   ```yaml
+   async_training:
+     partial_rollout: True
+     staleness_threshold: 0.5
+     # 其他async参数
+   ```
+4. **example**: 参考`recipe/fully_async_policy/shell/dapo_7b_async_retool.sh`
+
+### 实验结果
+
+为验证 `fully_async_policy` 在多轮工具调用任务中的性能，我们将其与标准 `colocate` 同步模式进行了对比。实验具体设置如下。
+
+* **SFT模型**: 实验基于 `Qwen2.5-7B-Instruct` 模型，使用`ReTool-SFT`数据集训练6个epoch；
+* **RL算法**: DAPO
+* **数据集**:
+    * 训练集: `DAPO-Math-17k`
+    * 测试集: `aime_2025`
+* **资源与模式对比**:
+    * `colocate sync`: 32卡 H20
+    * `fully_async_policy`: 16卡 Trainer + 16卡 Rollouter
+* **关键配置**:
+    1. **工具调用配置**:
+        * `multi_turn.enable: True`
+        * `multi_turn.max_user_turns: 16`
+        * `multi_turn.max_assistant_turns: 16`
+        * `multi_turn.tool_config_path: recipe/retool/sandbox_fusion_tool_config.yaml`
+    2. **`colocate sync`配置**:
+        * `ppo_mini_batch_size: 16`
+        * `train_batch_size: 64`
+    3. **`fully_async_policy`配置**:
+        * `ppo_mini_batch_size: 16`
+        * `trigger_parameter_sync_step: 4`
+        * `require_batches: 1`
+        * `staleness_threshold: 1`
+        * `partial_rollout: True`
+
+|    training mode   	| Resource allocation 	|   step  	|   gen   	| old_log_prob 	| update_actor 	| total time<br>100 step 	| total time<br>200 step 	|   aime_2025<br>acc/mean@30  	|
+|:------------------:	|:-------------------:	|:-------:	|:-------:	|:------------:	|:------------:	|:----------------------:	|:----------------------:	|:---------------------------:	|
+| colocate           	| 32                  	| 375.47  	|  228.03 	| 35.19        	| 111.84       	| 9h 46m                 	| 22h 28m                	| start:0.1078<br>last:0.2056   	|
+| fully_async_policy 	| 16: 16              	|  221.36 	| 40.59   	| \            	| 179.58       	| 6h 19m<br>(1.55x)      	| 14h 4m<br>(1.60x)      	| start:0.11<br>last:0.2044 	|
+
+> source data: https://wandb.ai/hou-zg-meituan/fully-async-policy-multiturn-tool?nw=nwuserhouzg
+
+## 后续计划
+
+* GRPO实验
+* megatron 适配
+* sglang 集成
+* transfer queue 集成
+* 异步参数同步
+* Areal异步算法实现
+* TPPO算法实现
+* 多轮及Tool的支持

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/agent_loop/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .agent_loop import FullyAsyncAgentLoopManager
+from .partial_single_turn_agent_loop import PartialSingleTurnAgentLoop
+from .partial_tool_agent_loop import AsyncPartialToolAgentLoop
+
+_ = [PartialSingleTurnAgentLoop, AsyncPartialToolAgentLoop]
+__all__ = [FullyAsyncAgentLoopManager]

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/agent_loop/agent_loop.py

@@ -0,0 +1,370 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import logging
+import os
+from typing import Any, Optional, Sequence
+
+import hydra
+import numpy as np
+import ray
+from omegaconf import DictConfig
+
+from verl.experimental.agent_loop.agent_loop import (
+    AgentLoopManager,
+    AgentLoopOutput,
+    AgentLoopWorker,
+    AsyncLLMServerManager,
+    DictConfigWrap,
+    _agent_loop_registry,
+    get_trajectory_info,
+)
+from verl.experimental.agent_loop.prometheus_utils import update_prometheus_config
+from verl.protocol import DataProto
+from verl.single_controller.ray import RayResourcePool, RayWorkerGroup
+from verl.utils.rollout_trace import (
+    rollout_trace_attr,
+    rollout_trace_op,
+)
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+class FullyAsyncLLMServerManager(AsyncLLMServerManager):
+    @rollout_trace_op
+    async def generate_for_partial(
+        self,
+        request_id,
+        *,
+        prompt_ids: list[int],
+        sampling_params: dict[str, Any],
+        image_data: Optional[list[Any]] = None,
+    ) -> tuple[list[Any], list[Any], Any] | tuple[Sequence[int], list[float], bool]:
+        """Generate tokens from prompt ids, used for async partial.
+
+        Args:
+            request_id (str): request id for sticky session.
+            prompt_ids (List[int]): List of prompt token ids.
+            sampling_params (Dict[str, Any]): Sampling parameters for the chat completion.
+
+        Returns:
+            output: A tuple representing the generation output.
+            - Element 0 (Sequence[int]): Generated response token IDs.
+            - Element 1 (list[float]): Log probabilities for the response token IDs.
+            - Element 2 (bool): A flag or status indicating cancellation.
+        """
+        server = self._choose_server(request_id)
+        output = await server.generate_for_partial.remote(
+            request_id=request_id,
+            prompt_ids=prompt_ids,
+            sampling_params=sampling_params,
+            image_data=image_data,
+        )
+        return output
+
+
+@ray.remote
+class FullyAsyncAgentLoopWorker(AgentLoopWorker):
+    def __init__(
+        self, config: DictConfig, server_handles: list[ray.actor.ActorHandle], reward_router_address: str = None
+    ):
+        self.server_manager = FullyAsyncLLMServerManager(config, server_handles)
+        super().__init__(config, server_handles, reward_router_address)
+        # A shared cancellation event for all agent loops running on this worker.
+        self.cancellation_event = asyncio.Event()
+
+    async def generate_sequences_no_post(
+        self, batch: DataProto, partial_output_list: Optional[list[AgentLoopOutput]]
+    ) -> tuple[list[AgentLoopOutput], bool] | tuple[DataProto, bool]:
+        """Generate sequences from agent loop.
+
+        Args:
+            batch (DataProto): Input batch.
+            partial_output_list: Optional[List[AgentLoopOutput]]: already rollout result.
+
+        Returns:
+            list[AgentLoopOutput]: List of agent loop outputs, one per sample in the batch.
+        """
+        config = self.config.actor_rollout_ref.rollout
+        sampling_params = dict(
+            temperature=config.temperature,
+            top_p=config.top_p,
+            repetition_penalty=1.0,
+            logprobs=config.calculate_log_probs,
+        )
+
+        # override sampling params for validation
+        if batch.meta_info.get("validate", False):
+            sampling_params["top_p"] = config.val_kwargs.top_p
+            sampling_params["temperature"] = config.val_kwargs.temperature
+
+        if "agent_name" not in batch.non_tensor_batch:
+            default_agent_loop = config.agent.default_agent_loop
+            batch.non_tensor_batch["agent_name"] = np.array([default_agent_loop] * len(batch), dtype=object)
+
+        if "index" in batch.non_tensor_batch:
+            index = batch.non_tensor_batch["index"]
+        else:
+            index = np.arange(len(batch))
+
+        trajectory_info = await get_trajectory_info(
+            batch.meta_info.get("global_steps", -1), index, batch.meta_info.get("validate", False)
+        )
+
+        if not partial_output_list:
+            partial_output_list = [None] * len(batch)
+        try:
+            tasks = []
+            for i in range(len(batch)):
+                kwargs = {k: v[i] for k, v in batch.non_tensor_batch.items()}
+                kwargs["output"] = partial_output_list[i]
+                tasks.append(
+                    asyncio.create_task(self._partial_run_agent_loop(sampling_params, trajectory_info[i], **kwargs))
+                )
+            outputs = await asyncio.gather(*tasks)
+        except Exception:
+            logger.exception("_partial_run_agent_loop failed")
+            raise
+
+        is_cancel = any(output.extra_fields.get("is_cancel", False) for output in outputs)
+        if not is_cancel:
+            output = self._postprocess(outputs)
+            output = self._addition_process(output)
+            return output, is_cancel
+        return outputs, is_cancel
+
+    def _addition_process(self, output: DataProto):
+        """collect metirics"""
+        metrics = output.meta_info.pop("metrics")  # List[Dict[str, str]]
+        processing_times_list = [item["generate_sequences"] for item in metrics]
+        tool_calls_times_list = [item["tool_calls"] for item in metrics]
+        output.non_tensor_batch["processing_times"] = processing_times_list
+        output.non_tensor_batch["tool_calls_times"] = tool_calls_times_list
+        return output
+
+    async def _partial_run_agent_loop(
+        self,
+        sampling_params: dict[str, Any],
+        trajectory: dict[str, Any],
+        *,
+        agent_name: str,
+        **kwargs,
+    ) -> AgentLoopOutput:
+        # Completed, return directly
+        if kwargs["output"] is not None and not kwargs["output"].extra_fields.get("is_cancel", False):
+            logger.info("In _partial_run_agent_loop, already completed, return derictly!")
+            return kwargs["output"]
+        try:
+            with rollout_trace_attr(
+                step=trajectory["step"],
+                sample_index=trajectory["sample_index"],
+                rollout_n=trajectory["rollout_n"],
+                validate=trajectory["validate"],
+                name="agent_loop",
+            ):
+                assert agent_name in _agent_loop_registry, (
+                    f"Agent loop {agent_name} not registered, registered agent loops: {_agent_loop_registry.keys()}"
+                )
+
+                agent_loop_config = _agent_loop_registry[agent_name]
+                agent_loop = hydra.utils.instantiate(
+                    config=agent_loop_config,
+                    trainer_config=DictConfigWrap(config=self.config),
+                    server_manager=self.server_manager,
+                    tokenizer=self.tokenizer,
+                    processor=self.processor,
+                    dataset_cls=self.dataset_cls,
+                    dataset_config=self.config.data,
+                )
+                output: AgentLoopOutput = await agent_loop.run(
+                    sampling_params, cancellation_event=self.cancellation_event, **kwargs
+                )
+                if not output.extra_fields.get("is_cancel", False):
+                    kwargs.pop("output", None)
+                    output = await self._agent_loop_postprocess(output, **kwargs)
+
+                return output
+        except Exception:
+            logger.exception("Agent_loop run failed")
+            raise
+
+    async def cancel_agent_loops(self):
+        """Set the shared cancellation event to stop all agent loops."""
+        self.cancellation_event.set()
+
+    async def resume_agent_loops(self):
+        """Clear the shared cancellation event."""
+        self.cancellation_event.clear()
+
+
+class FullyAsyncAgentLoopManager(AgentLoopManager):
+    def __init__(
+        self, config: DictConfig, worker_group: RayWorkerGroup = None, rm_resource_pool: RayResourcePool = None
+    ):
+        self.config = config
+        self.worker_group = worker_group
+        self.reward_model_manager = None
+        self.reward_router_address = None
+        self.agent_loop_workers_class = FullyAsyncAgentLoopWorker
+
+        # Select rollout replica class based on rollout name
+        rollout_name = config.actor_rollout_ref.rollout.name
+        if rollout_name == "sglang":
+            from verl.experimental.fully_async_policy.sglang_rollout.sglang_async_server import FullyAsyncSGLangReplica
+
+            self.rollout_replica_class = FullyAsyncSGLangReplica
+            print("[FullyAsyncAgentLoopManager] SGLang replica class selected")
+        elif rollout_name == "vllm":
+            from verl.experimental.fully_async_policy.vllm_rollout.vllm_async_server import FullyAsyncvLLMReplica
+
+            self.rollout_replica_class = FullyAsyncvLLMReplica
+            print("[FullyAsyncAgentLoopManager] vLLM replica class selected")
+        else:
+            raise ValueError(f"Unsupported rollout name: {rollout_name}. Supported values are 'sglang' and 'vllm'.")
+
+        self.rm_resource_pool = rm_resource_pool
+        self.rollout_replicas = None
+        self.server_handles = None
+        self.server_addresses = None
+        self.agent_loop_workers = None
+
+    @classmethod
+    async def create(
+        cls, config: DictConfig, worker_group: RayWorkerGroup = None, rm_resource_pool: RayResourcePool = None
+    ):
+        instance = cls(config, worker_group, rm_resource_pool)
+        await instance._async_init()
+        return instance
+
+    async def _async_init(self):
+        if self.config.reward_model.enable and self.config.reward_model.enable_resource_pool:
+            from verl.experimental.reward_loop import RewardModelManager
+
+            self.reward_model_manager = RewardModelManager(self.config.reward_model, self.rm_resource_pool)
+            self.reward_router_address = self.reward_model_manager.get_router_address()
+
+        await self._initialize_llm_servers_async()
+        self._init_agent_loop_workers()
+
+    async def _initialize_llm_servers_async(self):
+        rollout_world_size = (
+            self.config.actor_rollout_ref.rollout.tensor_model_parallel_size
+            * self.config.actor_rollout_ref.rollout.data_parallel_size
+            * self.config.actor_rollout_ref.rollout.pipeline_model_parallel_size
+        )
+        world_size = (
+            self.worker_group.world_size
+            if self.worker_group
+            else self.config.trainer.n_gpus_per_node * self.config.trainer.nnodes
+        )
+        num_replicas = world_size // rollout_world_size
+
+        rollout_config = self.config.actor_rollout_ref.rollout
+        model_config = self.config.actor_rollout_ref.model
+        self.rollout_replicas = [
+            self.rollout_replica_class(
+                replica_rank=replica_rank,
+                config=rollout_config,
+                model_config=model_config,
+                gpus_per_node=self.config.trainer.n_gpus_per_node,
+            )
+            for replica_rank in range(num_replicas)
+        ]
+
+        if self.worker_group:
+            await asyncio.gather(*[server.init_hybrid(self.worker_group) for server in self.rollout_replicas])
+        else:
+            await asyncio.gather(*[server.init_standalone() for server in self.rollout_replicas])
+
+        self.server_handles = [server._server_handle for server in self.rollout_replicas]
+        self.server_addresses = [server._server_address for server in self.rollout_replicas]
+
+        print(f"AgentLoopManager: {self.server_addresses}")
+        # Update Prometheus configuration with server addresses
+        if rollout_config.prometheus.enable:
+            if rollout_config.disable_log_stats:
+                raise ValueError("PROMETHEUS needs disable_log_stats==False, but it is currently True.")
+            await asyncio.to_thread(
+                update_prometheus_config, rollout_config.prometheus, self.server_addresses, rollout_config.name
+            )
+
+    async def generate_single_sample_async(
+        self,
+        sample: DataProto,
+        partial_output_list: Optional[list[AgentLoopOutput]],
+    ) -> tuple[list[AgentLoopOutput], bool] | tuple[DataProto, bool]:
+        """
+        Asynchronously process a single sample
+
+        Args:
+            sample: Single sample data
+            partial_output_list: Optional[List[AgentLoopOutput]]: already rollout result.
+
+        Returns:
+            list[AgentLoopOutput]: Processing results
+        """
+        worker = self._select_best_worker()
+        output_future = worker.generate_sequences_no_post.remote(sample, partial_output_list)
+        return await asyncio.wrap_future(output_future.future())
+
+    def _select_best_worker(self):
+        """Select the best worker, simple round-robin load balancing"""
+        if not hasattr(self, "_worker_index"):
+            self._worker_index = 0
+
+        worker = self.agent_loop_workers[self._worker_index]
+        self._worker_index = (self._worker_index + 1) % len(self.agent_loop_workers)
+        return worker
+
+    async def cancel(self):
+        worker_cancel_tasks = [worker.cancel_agent_loops.remote() for worker in self.agent_loop_workers]
+        rollout_cancel_tasks = [replica.cancel() for replica in self.rollout_replicas]
+        await asyncio.gather(*rollout_cancel_tasks, *worker_cancel_tasks)
+
+    async def resume(self):
+        rollout_resume_tasks = [replica.resume() for replica in self.rollout_replicas]
+        worker_resume_tasks = [worker.resume_agent_loops.remote() for worker in self.agent_loop_workers]
+        await asyncio.gather(*rollout_resume_tasks, *worker_resume_tasks)
+
+    async def wake_up(self):
+        await asyncio.gather(*[replica.wake_up() for replica in self.rollout_replicas])
+
+    async def sleep(self):
+        await asyncio.gather(*[replica.sleep() for replica in self.rollout_replicas])
+
+    async def reset_prefix_cache(self):
+        print("[FullyAsyncAgentLoopManager] Reset prefix cache ...")
+        # await asyncio.gather(*[replica.reset_prefix_cache() for replica in self.rollout_replicas])
+        # Note: debug
+        timeout = 5.0
+
+        async def reset_one(idx, replica):
+            print(f"[reset_prefix_cache] start replica={idx}")
+            try:
+                await asyncio.wait_for(replica.reset_prefix_cache(), timeout=timeout)
+            except asyncio.TimeoutError:
+                print(f"[reset_prefix_cache] TIMEOUT replica={idx} after {timeout}s")
+                return
+            except Exception as e:
+                print(f"[reset_prefix_cache] ERROR replica={idx}: {e!r}")
+                return
+            print(f"[reset_prefix_cache] done  replica={idx}")
+
+        tasks = [reset_one(i, replica) for i, replica in enumerate(self.rollout_replicas)]
+        await asyncio.gather(*tasks, return_exceptions=True)
+        print("[FullyAsyncAgentLoopManager] Reset prefix cache finished")
+
+    async def clear_kv_cache(self):
+        await asyncio.gather(*[replica.clear_kv_cache() for replica in self.rollout_replicas])

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/agent_loop/partial_single_turn_agent_loop.py

@@ -0,0 +1,115 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+from typing import Any, Optional
+from uuid import uuid4
+
+from verl.experimental.agent_loop import AgentLoopBase
+from verl.experimental.agent_loop.agent_loop import AgentLoopOutput, register
+from verl.utils.profiler import simple_timer
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+@register("partial_single_turn_agent")
+class PartialSingleTurnAgentLoop(AgentLoopBase):
+    """Naive agent loop that only do single turn chat completion."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.prompt_length = self.config.actor_rollout_ref.rollout.prompt_length
+        self.response_length = self.config.actor_rollout_ref.rollout.response_length
+        self.apply_chat_template_kwargs = self.config.data.get("apply_chat_template_kwargs", {})
+
+    async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput:
+        output: Optional[AgentLoopOutput] = kwargs.get("output", None)
+        messages = list(kwargs["raw_prompt"])
+        param_version = kwargs.get("param_version", 0)
+
+        metrics = {}
+        request_id = uuid4().hex
+        image_data = (kwargs.get("multi_modal_data") or {}).get("image", None)
+
+        param_version_start = param_version
+        param_version_end = param_version
+
+        if not output:
+            # TODO(baiyan): it is supposed to use the correct processor,
+            #    but I found the async training would hang if use_correct_processor=True.
+            #    so we use the tokenizer to tokenize the prompt for now.
+            use_correct_processor = False
+            if self.processor is not None and use_correct_processor:
+
+                def get_prompt_ids():
+                    raw_prompt = self.processor.apply_chat_template(
+                        messages,
+                        add_generation_prompt=True,
+                        tokenize=False,
+                        **self.apply_chat_template_kwargs,
+                    )
+                    model_inputs = self.processor(text=[raw_prompt], images=image_data, return_tensors="pt")
+                    return model_inputs.pop("input_ids").squeeze(0).tolist()
+
+                prompt_ids = await self.loop.run_in_executor(None, get_prompt_ids)
+            else:
+                prompt_ids = await self.loop.run_in_executor(
+                    None,
+                    lambda: self.tokenizer.apply_chat_template(
+                        messages, add_generation_prompt=True, tokenize=True, **self.apply_chat_template_kwargs
+                    ),
+                )
+        else:
+            if output.extra_fields.get("is_cancel", False):
+                # Resume the paused sample,
+                # add the result directly after prompt_ids,
+                # and reset generate_sequences metric
+                prompt_ids = output.prompt_ids + output.response_ids
+                metrics["generate_sequences"] = output.metrics.generate_sequences
+                param_version_start = output.extra_fields.get("param_version_start", param_version)
+            else:
+                # In the same batch of samples,
+                # some are canceled and some are not.
+                # The samples without partial rollout are returned directly.
+                return output
+        with simple_timer("generate_sequences", metrics):
+            response_ids, response_logprobs, is_cancel = await self.server_manager.generate_for_partial(
+                request_id=request_id, prompt_ids=prompt_ids, sampling_params=sampling_params, image_data=image_data
+            )
+        if not output:
+            response_mask = [1] * len(response_ids)
+        else:
+            # Pause the sample to be resumed, add the output result to response_ids, and reset response_mask
+            prompt_ids = output.prompt_ids
+            response_logprobs = output.response_logprobs + response_logprobs
+            response_ids = output.response_ids + response_ids
+            response_mask = [1] * len(response_ids)
+        if len(response_ids) >= self.response_length:
+            is_cancel = False
+
+        return AgentLoopOutput(
+            prompt_ids=prompt_ids,
+            response_ids=response_ids[: self.response_length],
+            response_mask=response_mask[: self.response_length],
+            response_logprobs=response_logprobs[: self.response_length],
+            num_turns=2,
+            metrics=metrics,
+            extra_fields={
+                "is_cancel": is_cancel,
+                "param_version_start": param_version_start,
+                "param_version_end": param_version_end,
+            },
+            # multi_modal_data={"image": image_data} if image_data is not None else {},
+        )

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/agent_loop/partial_tool_agent_loop.py

@@ -0,0 +1,281 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import copy
+import logging
+import os
+from typing import Any, Optional
+from uuid import uuid4
+
+from verl.experimental.agent_loop.agent_loop import AgentLoopOutput, register
+from verl.experimental.agent_loop.tool_agent_loop import AgentData, AgentState, ToolAgentLoop
+from verl.utils.profiler import simple_timer
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+@register("async_partial_tool_agent")
+class AsyncPartialToolAgentLoop(ToolAgentLoop):
+    """
+    Support for partial rollout with multiple tool invocations in Agent Loop
+
+    """
+
+    def __init__(self, trainer_config, **kwargs):
+        super().__init__(trainer_config, **kwargs)
+        self.enable_partial_rollout = trainer_config.config.async_training.get("partial_rollout", False)
+
+    # async def run(self, sampling_params: dict[str, Any], **kwargs) -> AgentLoopOutput:
+    async def run(
+        self, sampling_params: dict[str, Any], *, cancellation_event: asyncio.Event = None, **kwargs
+    ) -> AgentLoopOutput:
+        """
+        Main entrance, supports interruption/recovery
+
+        Args:
+            sampling_params: Sampling parameters
+            cancellation_event: cancellationn sginal
+            **kwargs: Contains output (for recovery), raw_prompt, param_version, etc.
+
+        Returns:
+            AgentLoopOutput: Include the is_cancel flag
+        """
+        param_version = kwargs.get("param_version", 0)
+        agent_data = None
+        state = None
+
+        # 1. check whether is the partial task
+        output: Optional[AgentLoopOutput] = kwargs.get("output", None)
+        if output and output.extra_fields.get("is_cancel", False):
+            agent_data, state = self._restore_from_output(output)
+
+            logger.info(f"[PartialToolAgent] Resuming from {state.value}")
+        else:
+            if output and not output.extra_fields.get("is_cancel", False):
+                # Completed, return directly
+                return output
+
+            agent_data = await self._init_agent_data(kwargs, param_version)
+            state = AgentState.PENDING
+            logger.info("[PartialToolAgent] Start from scratch")
+        # 2. run state machine
+        state = await self._run_state_machine(agent_data, state, sampling_params, cancellation_event)
+
+        # 3. bulid output
+        if state == AgentState.TERMINATED:
+            return self._build_completed_output(agent_data, param_version)
+        else:
+            # build cancelled output
+            return self._build_cancelled_output(agent_data, state)
+
+    async def _init_agent_data(self, kwargs: dict, param_version: int) -> AgentData:
+        messages = list(kwargs["raw_prompt"])
+        image_data = copy.deepcopy(kwargs.get("multi_modal_data", {}).get("image", None))
+        video_data = copy.deepcopy(kwargs.get("multi_modal_data", {}).get("video", None))
+        metrics = {}
+        request_id = uuid4().hex
+        tools_kwargs = kwargs.get("tools_kwargs", {})
+
+        # Initialize interaction if needed
+        interaction = None
+        interaction_kwargs = {}
+        if self.interaction_config_file:
+            interaction_kwargs = kwargs["extra_info"]["interaction_kwargs"]
+            if "name" not in interaction_kwargs:
+                raise ValueError("'name' key is required in interaction_kwargs")
+            interaction_name = interaction_kwargs["name"]
+            if interaction_name not in self.interaction_map:
+                raise ValueError(
+                    f"Interaction '{interaction_name}' not found in interaction_map. Available interactions: "
+                    f"{list(self.interaction_map.keys())}"
+                )
+            interaction = self.interaction_map[interaction_name]
+            await interaction.start_interaction(request_id, **interaction_kwargs)
+        # Create AgentData instance to encapsulate all state
+        agent_data = AgentData(
+            messages=messages,
+            image_data=image_data,
+            video_data=video_data,
+            metrics=metrics,
+            request_id=request_id,
+            tools_kwargs=tools_kwargs,
+            interaction=interaction,
+            interaction_kwargs=interaction_kwargs,
+        )
+
+        # additional param version record
+        agent_data.extra_fields["param_version_start"] = param_version
+        agent_data.extra_fields["param_version_end"] = param_version
+
+        return agent_data
+
+    def _restore_from_output(self, output: AgentLoopOutput) -> tuple[AgentData, AgentState]:
+        """restore AgentState and AgentData from output"""
+        agent_data = output.extra_fields.get("agent_data", None)
+        agent_state = output.extra_fields.get("agent_state", None)
+        if agent_data is None or agent_state is None:
+            raise ValueError(f"Unexpected situation: agent_data is {agent_data}, agent_state is {agent_state}")
+        return agent_data, agent_state
+
+    async def _run_state_machine(
+        self,
+        agent_data: AgentData,
+        state: AgentState,
+        sampling_params: dict[str, Any],
+        cancellation_event: asyncio.Event = None,
+    ) -> AgentState:
+        """
+        State machine.
+        Currently, interruptions are only supported to occur in the GENERATING state or other states have ended.
+        """
+        # State machine loop
+        while state != AgentState.TERMINATED:
+            if cancellation_event and cancellation_event.is_set():
+                logger.info(f"[PartialToolAgent] Cancellation detected. Interrupted before/at state: {state.value}")
+                return state
+            if state == AgentState.PENDING:
+                state = await self._handle_pending_state(agent_data, sampling_params)
+            elif state == AgentState.GENERATING:
+                state = await self._handle_generating_state_partial(agent_data, sampling_params)
+            elif state == AgentState.PROCESSING_TOOLS:
+                state = await self._handle_processing_tools_state(agent_data)
+            elif state == AgentState.INTERACTING:
+                state = await self._handle_interacting_state(agent_data)
+            else:
+                logger.error(f"[PartialToolAgent] Invalid state: {state}")
+                return AgentState.TERMINATED
+
+        return AgentState.TERMINATED
+
+    async def _handle_generating_state_partial(
+        self, agent_data: AgentData, sampling_params: dict[str, Any], ignore_termination: bool = False
+    ) -> AgentState:
+        """
+        Handle GENERATING state, support partial rollout
+        """
+        add_messages: list[dict[str, Any]] = []
+
+        with simple_timer("generate_sequences", agent_data.metrics):
+            # partial interface
+            if self.enable_partial_rollout:
+                response_ids, log_probs, is_cancel = await self.server_manager.generate_for_partial(
+                    request_id=agent_data.request_id,
+                    prompt_ids=agent_data.prompt_ids,
+                    sampling_params=sampling_params,
+                    image_data=agent_data.image_data,
+                )
+
+                if is_cancel:
+                    # Save the generated parts
+                    agent_data.response_ids = response_ids
+                    agent_data.prompt_ids += agent_data.response_ids
+                    agent_data.response_mask += [1] * len(response_ids)
+                    if log_probs:
+                        agent_data.response_logprobs += log_probs
+                    if not ignore_termination and len(agent_data.response_mask) >= self.response_length:
+                        # If response_length has reached the limit,
+                        # it is considered to have ended normally.
+                        agent_data.assistant_turns += 1
+                        return AgentState.TERMINATED
+                    return AgentState.GENERATING
+            else:
+                # original generate interface
+                output = await self.server_manager.generate(
+                    request_id=agent_data.request_id,
+                    prompt_ids=agent_data.prompt_ids,
+                    sampling_params=sampling_params,
+                    image_data=agent_data.image_data,
+                )
+                response_ids = output.token_ids
+                log_probs = output.log_probs
+
+        agent_data.assistant_turns += 1
+        agent_data.response_ids = response_ids
+        agent_data.prompt_ids += agent_data.response_ids
+        agent_data.response_mask += [1] * len(agent_data.response_ids)
+        if log_probs:
+            agent_data.response_logprobs += log_probs
+
+        if not ignore_termination and len(agent_data.response_mask) >= self.response_length:
+            return AgentState.TERMINATED
+        if self.max_assistant_turns and agent_data.assistant_turns >= self.max_assistant_turns:
+            return AgentState.TERMINATED
+        if self.max_user_turns and agent_data.user_turns >= self.max_user_turns:
+            return AgentState.TERMINATED
+
+        # Extract tool calls
+        _, agent_data.tool_calls = await self.tool_parser.extract_tool_calls(agent_data.response_ids)
+
+        # Handle interaction if needed
+        if self.interaction_config_file:
+            assistant_message = await self.loop.run_in_executor(
+                None, lambda: self.tokenizer.decode(agent_data.response_ids, skip_special_tokens=True)
+            )
+            add_messages.append({"role": "assistant", "content": assistant_message})
+            agent_data.messages.extend(add_messages)
+
+        # Determine next state
+        if agent_data.tool_calls:
+            return AgentState.PROCESSING_TOOLS
+        elif self.interaction_config_file:
+            return AgentState.INTERACTING
+        else:
+            return AgentState.TERMINATED
+
+    def _build_completed_output(self, agent_data: AgentData, param_version: int) -> AgentLoopOutput:
+        """build completed output"""
+        response_ids = agent_data.prompt_ids[-len(agent_data.response_mask) :]
+        prompt_ids = agent_data.prompt_ids[: len(agent_data.prompt_ids) - len(agent_data.response_mask)]
+        multi_modal_data = {"image": agent_data.image_data} if agent_data.image_data is not None else {}
+        output = AgentLoopOutput(
+            prompt_ids=prompt_ids,
+            response_ids=response_ids[: self.response_length],
+            response_mask=agent_data.response_mask[: self.response_length],
+            multi_modal_data=multi_modal_data,
+            response_logprobs=agent_data.response_logprobs[: self.response_length]
+            if agent_data.response_logprobs
+            else None,
+            num_turns=agent_data.user_turns + agent_data.assistant_turns + 1,
+            metrics=agent_data.metrics,
+            extra_fields={},
+        )
+        output.extra_fields.update(
+            {
+                "turn_scores": agent_data.turn_scores,
+                "tool_rewards": agent_data.tool_rewards,
+                "is_cancel": False,
+                "param_version_start": agent_data.extra_fields["param_version_start"],
+                "param_version_end": param_version,
+            }
+        )
+        return output
+
+    def _build_cancelled_output(self, agent_data: AgentData, state: AgentState) -> AgentLoopOutput:
+        """build cancelled output"""
+        return AgentLoopOutput(
+            prompt_ids=[],
+            response_ids=[],
+            response_mask=[],
+            multi_modal_data={},
+            response_logprobs=None,
+            num_turns=0,
+            metrics=agent_data.metrics,
+            extra_fields={
+                "is_cancel": True,
+                "agent_data": agent_data,
+                "agent_state": state,
+            },
+        )

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/base_detach_sync.py

@@ -0,0 +1,238 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import logging
+import os
+import threading
+
+import torch
+from omegaconf import DictConfig
+from ray.util.collective import collective
+
+from verl.single_controller.base.decorator import Dispatch, register
+from verl.utils.device import get_torch_device, is_npu_available
+from verl.utils.distributed import stateless_init_process_group
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+class BaseDetachNcclSync:
+    _bucket_size_mb = 1024.0
+    _sync_history = []
+    _max_history_size = 20
+    _last_avg_bucket_size = 1024.0
+
+    def __init__(self, config: DictConfig, role: str):
+        self._bg_loop = asyncio.new_event_loop()
+        self._bg_thread = threading.Thread(
+            target=self._start_background_loop, args=(self._bg_loop,), name="rollout_actor_async_worker", daemon=True
+        )
+        self._bg_thread.start()
+        logger.info(f"[DetachNcclSync] Background thread for SGLang sync started. PID: {os.getpid()}")
+
+    @classmethod
+    def get_bucket_size_mb(cls):
+        return cls._bucket_size_mb
+
+    @classmethod
+    def get_last_avg_bucket_size(cls):
+        return cls._last_avg_bucket_size
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=True)
+    def get_last_avg_bucket_size_remote(self):
+        return BaseDetachNcclSync._last_avg_bucket_size
+
+    @classmethod
+    def record_sync_metrics(cls, bucket_size_mb, sync_time):
+        """Dynamically adjust the bucket size based on past synchronization times."""
+        bucket_size_mb_value = bucket_size_mb[0] if isinstance(bucket_size_mb, list) else bucket_size_mb
+        print(f"[DetachNcclSync] sync_metrics: bucket_size_mb={bucket_size_mb_value:.2f}MB, sync_time={sync_time:.2f}s")
+        cls._sync_history.append((bucket_size_mb_value, sync_time))
+        if len(cls._sync_history) > cls._max_history_size:
+            cls._sync_history.pop(0)
+
+        MIN_BUCKET_SIZE_MB = 512
+        MAX_BUCKET_SIZE_MB = 8192  # 8GB
+
+        if len(cls._sync_history) < 4:
+            cls._bucket_size_mb = min(MAX_BUCKET_SIZE_MB, cls._bucket_size_mb * 1.5)
+        else:
+            times = [t for _, t in cls._sync_history]
+            buckets = [b for b, _ in cls._sync_history]
+            recent_avg_time = sum(times[-2:]) / 2
+            previous_avg_time = sum(times[-4:-2]) / 2
+            recent_avg_bucket = sum(buckets[-2:]) / 2
+            previous_avg_bucket = sum(buckets[-4:-2]) / 2
+
+            performance_improved = recent_avg_time < previous_avg_time
+            bucket_increased = recent_avg_bucket > previous_avg_bucket
+            time_change_ratio = (
+                abs(recent_avg_time - previous_avg_time) / previous_avg_time if previous_avg_time > 0 else 0.0
+            )
+
+            if time_change_ratio > 0.2:
+                increase_step, decrease_step = 1.2, 0.8
+            elif time_change_ratio > 0.1:
+                increase_step, decrease_step = 1.1, 0.9
+            elif time_change_ratio > 0.05:
+                increase_step, decrease_step = 1.05, 0.95
+            else:
+                increase_step, decrease_step = 1.02, 0.98
+
+            should_increase = (performance_improved and bucket_increased) or (
+                not performance_improved and not bucket_increased
+            )
+            step = increase_step if should_increase else decrease_step
+            new_size = cls._bucket_size_mb * step
+            cls._bucket_size_mb = min(MAX_BUCKET_SIZE_MB, max(MIN_BUCKET_SIZE_MB, new_size))
+
+    def _start_background_loop(self, loop):
+        asyncio.set_event_loop(loop)
+        try:
+            loop.run_forever()
+        except Exception as e:
+            logger.error(f"[DetachNcclSync] Background loop crashed: {e}")
+
+    def _run_async_safely(self, coro):
+        if not self._bg_thread.is_alive():
+            raise RuntimeError("Background thread for SGLang sync is not running!")
+
+        future = asyncio.run_coroutine_threadsafe(coro, self._bg_loop)
+        return future.result()
+
+    def __del__(self):
+        if hasattr(self, "_bg_loop") and self._bg_loop.is_running():
+            self._bg_loop.call_soon_threadsafe(self._bg_loop.stop)
+        if hasattr(self, "_bg_thread") and self._bg_thread.is_alive():
+            self._bg_thread.join(timeout=1.0)
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False)
+    def init_checkpoint_engine(self, rank_offset: int, actor_num: int, rollout_num: int):
+        from .checkpoint_engine import CheckpointEngine
+
+        current_rank = torch.distributed.get_rank() + rank_offset
+        actor_ranks = list(range(actor_num))
+        rollout_ranks = [rank + actor_num for rank in range(rollout_num)]
+        assert rank_offset == 0 or rank_offset == actor_num
+
+        self.checkpoint_engine = CheckpointEngine(
+            current_rank, actor_ranks, rollout_ranks, self.config.checkpoint_engine.device_buffer_size_M
+        )
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False)
+    def create_weight_sync_group(self, master_address, master_port, rank_offset, world_size):
+        rank = torch.distributed.get_rank() + rank_offset
+        self._weight_sync_group = stateless_init_process_group(
+            master_address,
+            master_port,
+            rank,
+            world_size,
+            get_torch_device().current_device(),
+        )
+
+    @staticmethod
+    def get_inference_model(rollout):
+        """
+        Get models according to different types of inference_engine
+        Args:
+            rollout: rollout object
+        Returns:
+            model: model object (for vllm) or rollout object itself (for sglang)
+        """
+        inference_engine = rollout.inference_engine
+        if hasattr(inference_engine, "llm_engine"):
+            inference_model = inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner.model
+        elif hasattr(inference_engine, "worker"):
+            inference_model = inference_engine.worker.model_runner.model
+        else:
+            raise AttributeError(
+                f"Unsupported inference_engine type: {type(inference_engine)}. "
+                f"Expected LLM (with llm_engine attribute) or WorkerWrapperBase (with worker attribute)."
+            )
+        return inference_model
+
+    def _sync_sglang_weights(self, inference_model, params, sync_group_name):
+        bucket_size_bytes = int(self.get_bucket_size_mb() * 1024 * 1024)
+        actual_bucket_sizes = []
+        current_batch = []
+        current_batch_size = 0
+
+        def flush_batch():
+            if current_batch:
+                actual_bucket_sizes.append(current_batch_size / (1024 * 1024))
+                self._run_async_safely(self.update_weights(inference_model, iter(current_batch)))
+                get_torch_device().synchronize()
+                current_batch.clear()
+
+        for key, shape, dtype in self._weights_info:
+            tensor = torch.empty(shape, dtype=dtype, device=get_torch_device().current_device())
+            if self._is_actor:
+                assert key in params
+                origin_data = params[key]
+                if hasattr(origin_data, "full_tensor"):
+                    origin_data = origin_data.full_tensor()
+                if torch.distributed.get_rank() == 0:
+                    tensor.copy_(origin_data)
+            collective.broadcast(tensor, src_rank=0, group_name=sync_group_name)
+
+            tensor_size = tensor.numel() * tensor.element_size()
+            current_batch.append((key, tensor))
+            current_batch_size += tensor_size
+
+            if current_batch_size >= bucket_size_bytes:
+                flush_batch()
+                current_batch_size = 0
+
+        flush_batch()
+        cls = type(self)
+        cls._last_avg_bucket_size = (
+            sum(actual_bucket_sizes) / len(actual_bucket_sizes) if actual_bucket_sizes else self.get_bucket_size_mb()
+        )
+
+        # Resume kv_cache after weights sync to restore GPU memory released during pause
+        if self._is_rollout and self.rollout_device_mesh["infer_tp"].get_local_rank() == 0:
+            self._run_async_safely(inference_model.resume_memory_occupation(tags=["kv_cache"]))
+
+    def _sync_vllm_weights(self, inference_model, params, sync_group_name):
+        for key, shape, dtype in self._weights_info:
+            tensor = torch.empty(shape, dtype=dtype, device=get_torch_device().current_device())
+            if self._is_actor:
+                assert key in params
+                origin_data = params[key]
+                if hasattr(origin_data, "full_tensor"):
+                    origin_data = origin_data.full_tensor()
+                if torch.distributed.get_rank() == 0:
+                    tensor.copy_(origin_data)
+            if is_npu_available:
+                self._weight_sync_group.broadcast(tensor, src=0, stream=get_torch_device().current_stream())
+            else:
+                collective.broadcast(tensor, src_rank=0, group_name=sync_group_name)
+            if self._is_rollout:
+                inference_model.load_weights([(key, tensor)])
+
+    async def update_weights(self, inference_engine, params):
+        from sglang.srt.weight_sync.utils import update_weights as sgl_update_weights
+
+        await sgl_update_weights(
+            engine=inference_engine,
+            params_batch=params,
+            device_mesh_key="infer_tp",
+            device_mesh=self.rollout_device_mesh,
+        )
+
+        if self.rollout_device_mesh["infer_tp"].get_local_rank() == 0:
+            await inference_engine.flush_cache()

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/checkpoint_engine.py

@@ -0,0 +1,522 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This logic is largely copied from:
+- https://github.com/MoonshotAI/checkpoint-engine
+"""
+
+import concurrent.futures
+import os
+import re
+import socket
+import subprocess
+import threading
+from collections.abc import Callable
+from functools import lru_cache
+from typing import TYPE_CHECKING, Annotated, Any, TypedDict
+
+import torch
+import zmq
+from pydantic import BaseModel, PlainSerializer, PlainValidator, WithJsonSchema
+from ray.util.collective import collective
+
+from verl.utils.device import (
+    get_device_name,
+    get_torch_device,
+)
+
+if TYPE_CHECKING:
+    from typing import TypeVar
+
+    from typing_extensions import TypedDict
+
+    class FileMeta(TypedDict):
+        key: str  # parameter name
+        dtype: torch.dtype
+        shape: torch.Size
+        type: type
+        tp_concat_dim: int
+
+    T = TypeVar("T")
+
+
+def _dt_validate(value: Any) -> torch.dtype:
+    """Validate the input value to ensure it is a valid torch.dtype"""
+    if isinstance(value, str):
+        if not value.startswith("torch."):
+            raise ValueError(f"dtype {value} should start with torch.")
+        try:
+            value = getattr(torch, value.split(".")[1])
+        except AttributeError as e:
+            raise ValueError(f"unknown dtype: {value}") from e
+    if not isinstance(value, torch.dtype):
+        raise TypeError(f"dtype {value} should be torch.dtype, got {type(value)}")
+    return value
+
+
+# Annotated type for torch.dtype with validation and serialization
+_TorchDtype = Annotated[
+    torch.dtype,
+    PlainValidator(_dt_validate),
+    PlainSerializer(lambda x: str(x), return_type=str),
+    WithJsonSchema({"type": "string"}, mode="serialization"),
+]
+
+
+def _size_validate(value: Any) -> torch.Size:
+    """Validate the input value to ensure it is a valid torch.Size"""
+    if isinstance(value, list | tuple):
+        return torch.Size(value)
+    if not isinstance(value, torch.Size):
+        raise TypeError(f"size {value} should be torch.Size, got {type(value)}")
+    return value
+
+
+# Annotated type for torch.Size with validation and serialization
+_TorchSize = Annotated[
+    torch.Size,
+    PlainValidator(_size_validate),
+    PlainSerializer(lambda x: tuple(x), return_type=tuple),
+    WithJsonSchema({"type": "array", "items": {"type": "integer"}}, mode="serialization"),
+]
+
+
+def _tensor_validate(value: Any) -> torch.Tensor:
+    """Validate the input value to ensure it is a valid torch.Tensor"""
+    if isinstance(value, torch.Tensor):
+        return value
+    raise TypeError(f"tensor {value} should be torch.Tensor, got {type(value)}")
+
+
+# Annotated type for torch.Tensor with validation
+_TorchTensor = Annotated[
+    torch.Tensor,
+    PlainValidator(_tensor_validate),
+]
+
+
+class ParameterMeta(BaseModel):
+    """Metadata for a parameter including name, dtype, and shape"""
+
+    name: str
+    dtype: _TorchDtype
+    shape: _TorchSize
+
+
+class MemoryBuffer(BaseModel):
+    """
+    MemoryBuffer assembles a group of parameter tensors into a single buffer,
+    and records the meta information of each original parameter.
+    """
+
+    buffer: _TorchTensor
+    size: int  # size of buffer in bytes
+    metas: list[ParameterMeta]
+
+
+class MemoryBufferMeta(BaseModel):
+    """The meta info of MemoryBuffer, but not store the buffer data"""
+
+    size: int
+    metas: list[ParameterMeta]
+
+
+# 256 bytes alignment when flatten torch tensors to uint8 buffer
+_ALIGN_SIZE = 256
+
+
+def _align_size(dtype: torch.dtype, shape: torch.Size) -> int:
+    """
+    Calculate the aligned size of a torch tensor
+
+    If the tensor's size (in bytes) cannot be evenly divided by _ALIGN_SIZE,
+    it will be rounded up to the nearest multiple of _ALIGN_SIZE.
+
+    Args:
+        dtype (torch.dtype): The data type of the tensor (e.g., torch.float32, torch.int64).
+        shape (torch.Size): The shape of the tensor, representing its dimensions.
+
+    Returns:
+        int: The aligned size of the tensor in bytes.
+    """
+    return (dtype.itemsize * shape.numel() + _ALIGN_SIZE - 1) // _ALIGN_SIZE * _ALIGN_SIZE
+
+
+@lru_cache(maxsize=1)
+def get_ip() -> str:
+    try:
+        # try to get ip from network interface
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+            s.connect(("8.8.8.8", 80))
+            return s.getsockname()[0]
+    except Exception as e:  # noqa: BLE001
+        # fallback to get ip from hostname
+        print(f"fail to get ip from network interface, fallback to get ip from hostname: {e}")
+        return socket.gethostbyname(socket.gethostname())
+
+
+def npu_generate_uuid() -> str:
+    """Generate uuid for each npu device"""
+    str_pid = str(os.getpid())
+    npu_num = 8
+    try:
+        for npu_id in range(npu_num):
+            cmd = ["npu-smi", "info", "-t", "proc-mem", "-i", str(npu_id)]
+            result = subprocess.run(cmd, check=True, capture_output=True, text=True)  # noqa: S603
+            str_result = str(result.stdout)
+            if str_pid in str_result:
+                # In A3 server, one NPU has two chips.
+                match_chip_count = re.search(r"Chip Count[^\d]*(\d+)", str_result)
+                chip_count = int(match_chip_count.group(1))
+                search_after_pid = str_result[str_result.find(str_pid) + len(str_pid) :]
+                match_chip_id = re.search(r"Chip ID[^\d]*(\d+)", search_after_pid)
+                chip_id = int(match_chip_id.group(1))
+                return f"{get_ip()}-{npu_id * chip_count + chip_id}"
+        raise ValueError("The current process is not running on the npu device")
+    except subprocess.CalledProcessError as e:
+        raise ValueError("The current process is not running on the npu device") from e
+
+
+def _get_physical_device_id(device_index: int | None = None) -> str:
+    """
+    Get the physical device (GPU or NPU) uuid of the current device
+    """
+    try:
+        if get_device_name() == "npu":
+            return f"NPU-{npu_generate_uuid()}"
+        else:
+            return f"GPU-{get_torch_device().get_device_properties(device_index).uuid!s}"
+    except AssertionError as e:
+        raise ValueError(f"fail to get physical gpu id {device_index}") from e
+
+
+class FlattenedTensorMetadata(TypedDict):
+    name: str
+    shape: torch.Size
+    dtype: torch.dtype
+    # specify the start offset of this tensor in shared ipc_buffer tensor
+    offset: int
+
+
+def _to_flattened_tensor_meta(metas: list[ParameterMeta], offset: int = 0) -> list[FlattenedTensorMetadata]:
+    """
+    compute the offset of each parameter in the buffer
+
+    Args:
+        metas (list[ParameterMeta]): The list of parameter metas info
+        offset (int): The start offset of the buffer. Defaults to 0.
+
+    Returns:
+        list[FlattenedTensorMetadata]: The list of FlattenedTensorMetadata:
+    """
+    ret = []
+    for meta in metas:
+        size = _align_size(meta.dtype, meta.shape)
+        ret.append(
+            {
+                "name": meta.name,
+                "dtype": meta.dtype,
+                "shape": meta.shape,
+                "offset": offset,
+            }
+        )
+        offset += size
+    return ret
+
+
+def _extract_weights(
+    flatten_metas: list[FlattenedTensorMetadata], buffer: torch.Tensor
+) -> list[tuple[str, torch.Tensor]]:
+    """
+    According to the flatten_metas and buffer, extract the weights
+    """
+
+    assert buffer is not None
+    weights: list[tuple[str, torch.Tensor]] = []
+    for item in flatten_metas:
+        shape = item["shape"]
+        if isinstance(shape, list | tuple):
+            shape = torch.Size(shape)
+        assert isinstance(shape, torch.Size)
+        dtype, offset = item["dtype"], item["offset"]
+        size = dtype.itemsize * shape.numel()
+        tensor = buffer[offset : offset + size].view(dtype=dtype).view(shape)
+        weights.append((item["name"], tensor))
+    return weights
+
+
+class CheckpointEngine:
+    """
+    CheckpointEngine class for control parameters synchronization.
+    Each trainer/rollout rank has a CheckpointEngine instance.
+    """
+
+    def __init__(
+        self, current_rank: int, actor_ranks: list[int], rollout_ranks: list[int], device_buffer_size_M: int
+    ) -> None:
+        self.current_rank = current_rank
+        self.actor_ranks = actor_ranks
+        self.rollout_ranks = rollout_ranks
+        # global_buckets saves the global MemoryBufferMeta infos.
+        # Thus each CheckpointEngine instance can control their operations in SPMD
+        self.global_buckets: dict[int, list[MemoryBufferMeta]] = None
+        # min device_buffer_size for h2d and broadcast
+        self.device_buffer_size_M = device_buffer_size_M
+
+        # ipc config for broadcast in pipeline mode
+        self._zmq_ctx = zmq.Context()
+        self._zmq_addr_counter: int = 0
+        device_index = self.current_rank % get_torch_device().device_count()
+        self._device_uuid = _get_physical_device_id(device_index)
+
+    def register_checkpoint(
+        self, weights_info: list[tuple[str, torch.Size, torch.dtype]], cpu_named_params: dict[str, torch.Tensor]
+    ):
+        """
+        Register checkpoint information and prepare memory buffers for parameter synchronization.
+
+        This function organizes the parameters into memory buckets for efficient synchronization
+        and prepares pinned memory buffers for faster data transfer between CPU and device.
+
+        Args:
+            weights_info (list[tuple[str, torch.Size, torch.dtype]]):
+                A list of tuples containing parameter name, shape, and data type.
+            cpu_named_params (dict[str, torch.Tensor]):
+                A dictionary mapping parameter names to their corresponding CPU tensors.
+
+        Steps:
+            1. Calculate the bucket size based on the largest parameter tensor size and the device buffer size.
+            2. Organize parameters into global buckets for each actor rank, ensuring that the total size of each bucket
+               does not exceed the bucket size.
+            3. For actor ranks, allocate pinned memory buffers for each bucket and copy the parameter tensors
+               into these buffers.
+
+        Notes:
+            Each CheckpointEngine instance maintains the global buckets metas,
+            but stores part of parmas data in host memory
+        """
+        bucket_size = max(
+            self.device_buffer_size_M << 20, max(_align_size(dtype, shape) for _, shape, dtype in weights_info)
+        )
+        print(
+            f"set checkpoint_engine device buffer size: {self.device_buffer_size_M}M, "
+            f"and finally set it to {bucket_size >> 20}M considering the largest parameter tensor size"
+        )
+        self.bucket_size = bucket_size
+
+        # global_buckets saves the global MemoryBufferMeta infos.
+        if self.global_buckets is None:
+            self.global_buckets = {rank: [MemoryBufferMeta(size=0, metas=[])] for rank in self.actor_ranks}
+
+            actor_ranks_size = len(self.actor_ranks)
+            assert actor_ranks_size > 0, f"actor_ranks:{self.actor_ranks} should not be empty"
+            for param_idx, (param_name, param_shape, param_dtype) in enumerate(weights_info):
+                # Each parameter is assigned to an actor rank, and only this rank will store it
+                assgin_rank = self.actor_ranks[param_idx % actor_ranks_size]
+                param_size = _align_size(param_dtype, param_shape)
+
+                if self.global_buckets[assgin_rank][-1].size + param_size > bucket_size:
+                    assert self.global_buckets[assgin_rank][-1].size, (
+                        f"global_buckets[{assgin_rank}][-1].size:{self.global_buckets[assgin_rank][-1].size}"
+                        " should not be 0"
+                    )
+                    self.global_buckets[assgin_rank].append(MemoryBufferMeta(size=0, metas=[]))
+                self.global_buckets[assgin_rank][-1].metas.append(
+                    ParameterMeta(name=param_name, dtype=param_dtype, shape=param_shape)
+                )
+                self.global_buckets[assgin_rank][-1].size += param_size
+
+        def register_pin_memory(idx: int, size: int) -> tuple[int, torch.Tensor]:
+            """Allocate pinned memory for a bucket."""
+            buffer = torch.empty(size, dtype=torch.uint8, pin_memory=True)
+            return idx, buffer
+
+        def register_tensor(buffer: torch.Tensor, offset: int, tensor: torch.Tensor):
+            """Copy a tensor into a pinned memory buffer."""
+            buffer[offset : offset + tensor.nbytes] = tensor.view(-1).view(dtype=torch.uint8)
+
+        memory_buffers = []  # for rollout rank, return empty buffer
+        if self.current_rank in self.actor_ranks:  # is_actor
+            local_buckets = self.global_buckets[self.current_rank]
+            memory_buffers = [
+                MemoryBuffer(buffer=torch.empty(0), size=bucket.size, metas=bucket.metas) for bucket in local_buckets
+            ]
+
+            # Use thread pool to accelerate organize parameters into buckets
+            with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor:
+                futures = [
+                    executor.submit(register_pin_memory, idx, bucket.size) for idx, bucket in enumerate(local_buckets)
+                ]
+                new_futures = []
+                for future in concurrent.futures.as_completed(futures):
+                    idx, buffer = future.result()
+                    assert buffer.numel() == local_buckets[idx].size, (
+                        f"buffer numel {buffer.numel()} should be equal to bucket size {local_buckets[idx].size}"
+                    )
+                    memory_buffers[idx].buffer = buffer
+                    print(
+                        f"[rank{self.current_rank}] register pin_memory for "
+                        f" bucket {idx + 1}/{len(local_buckets)} finished, "
+                        f"size {buffer.numel() / 1024 / 1024:.2f}MiB, start to copy tensors to buffer"
+                    )
+                    offset = 0
+                    for meta in local_buckets[idx].metas:
+                        name = meta.name
+                        tensor = cpu_named_params[name]
+                        size = _align_size(tensor.dtype, tensor.shape)
+                        assert size == _align_size(meta.dtype, meta.shape), (
+                            f"tensor {name} size {size} should be equal to "
+                            f"meta size {_align_size(meta.dtype, meta.shape)}"
+                        )
+                        new_futures.append(executor.submit(register_tensor, buffer, offset, tensor))
+                        offset += size
+                for future in concurrent.futures.as_completed(new_futures):
+                    future.result()
+
+        self.memory_buffers = memory_buffers
+
+    def get_max_buckets_num_per_rank(self):
+        """
+        Get the maximum number of buckets for all rank.
+        """
+        assert self.global_buckets is not None
+        return max(len(buckets) for buckets in self.global_buckets.values())
+
+    def _bind_zmq_socket(self) -> tuple[zmq.Socket, list[tuple[str, str]]]:
+        """
+        Bind zmq socket for broadcast.
+        """
+
+        def zmq_handle(device_uuid: str) -> str:
+            return f"ipc://@checkpoint-engine-{device_uuid}-{self._zmq_addr_counter}.sock"
+
+        socket_path = zmq_handle(self._device_uuid)
+        socket = self._zmq_ctx.socket(zmq.REQ)
+        socket.bind(socket_path)
+        self._zmq_addr_counter += 1
+        return socket, socket_path
+
+    def update_checkpoint(self, inference_model, group_name: str, overlap_broadcast_and_consume: bool = False):
+        """
+        Update the checkpoint by broadcasting and loading weights.
+
+        This function handles the synchronization of parameters across ranks by:
+        1. Copying data from memory buffers to device buffers (h2d_buffer).
+        2. Broadcasting the data to all ranks using collective communication.
+        3. Loading the weights into the inference model if provided.
+        4. Optionally, use a pipeline approach for broadcasting and loading weights.
+
+        Args:
+            inference_model: The model to load weights into. If None (trainer rank), weights are only broadcasted.
+            group_name (str): The name of the collective communication group.
+            overlap_broadcast_and_consume (bool): Whether to use the pipeline approach
+            for broadcasting and loading weights.
+        """
+        try:
+            h2d_buffer: torch.Tensor | None = (
+                None
+                if self.current_rank in self.rollout_ranks
+                else torch.empty(self.bucket_size, dtype=torch.uint8, device=get_torch_device().current_device())
+            )
+            # for pipeline mode, we need to allocate 2x buffer size
+            broadcast_load_buffer = torch.empty(
+                self.bucket_size * (2 if overlap_broadcast_and_consume else 1),
+                dtype=torch.uint8,
+                device=get_torch_device().current_device(),
+            )
+        except Exception:
+            print(
+                "allocate buffer for update_checkpoint failed, "
+                "you may need to reduce "
+                "config.async_training.checkpoint_engine.device_buffer_size_M"
+            )
+            raise
+
+        max_h2d_iter = self.get_max_buckets_num_per_rank()
+
+        if overlap_broadcast_and_consume:
+            socket, socket_path = self._bind_zmq_socket()
+
+            # Define a function to update weights from IPC
+            def update_weights_from_ipc_(socket_path):
+                zmq_ctx = zmq.Context()
+                socket = zmq_ctx.socket(zmq.REP)
+                socket.connect(socket_path)
+                socket.recv_pyobj()
+                socket.send(b"")
+
+                while True:
+                    payload: tuple[Callable, tuple] | list[FlattenedTensorMetadata] | None = socket.recv_pyobj()
+                    if payload is None:
+                        # means the update is done
+                        get_torch_device().synchronize()
+                        socket.send(b"")
+                        break
+                    assert isinstance(payload, list)
+                    if inference_model is not None:
+                        inference_model.load_weights(_extract_weights(payload, broadcast_load_buffer))
+                    get_torch_device().synchronize()
+                    socket.send(b"")
+
+            req_thread = threading.Thread(
+                target=update_weights_from_ipc_,
+                args=(socket_path,),
+            )
+            req_thread.start()
+            socket.send_pyobj(b"")
+            get_torch_device().synchronize()
+
+        gidx = 0
+        local_buckets = self.global_buckets.get(self.current_rank, [])
+
+        for i in range(max_h2d_iter):
+            # Step 1: Each actor rank copy the parameter tensor into device memory
+            if i < len(self.memory_buffers):
+                h2d_buffer[: local_buckets[i].size].data.copy_(self.memory_buffers[i].buffer)
+
+            # Step 2: Broadcast the device data in turn
+            for broadcast_rank, _buckets in self.global_buckets.items():
+                if i >= len(_buckets):
+                    continue
+                bucket = _buckets[i]
+
+                # Prepare the broadcast buffer
+                start = gidx % 2 * self.bucket_size if overlap_broadcast_and_consume else 0
+                buffer_b: torch.Tensor = broadcast_load_buffer[start : start + bucket.size]
+                if broadcast_rank == self.current_rank:
+                    buffer_b.data.copy_(h2d_buffer[: bucket.size])
+
+                # Broadcast the buffer to all ranks
+                collective.broadcast(buffer_b, src_rank=broadcast_rank, group_name=group_name)
+
+                if overlap_broadcast_and_consume:
+                    socket.recv()
+                    collective.barrier(group_name=group_name)
+                    socket.send_pyobj(_to_flattened_tensor_meta(bucket.metas, start))
+                elif inference_model is not None:
+                    named_tensor = _to_flattened_tensor_meta(bucket.metas, 0)
+                    inference_model.load_weights(_extract_weights(named_tensor, buffer_b))
+
+                gidx += 1
+
+        if overlap_broadcast_and_consume:
+            socket.recv()
+            socket.send_pyobj(None)
+            socket.recv()
+            req_thread.join()
+            socket.close()
+
+        collective.barrier(group_name=group_name)
+        # clear host memory cache
+        self.memory_buffers = []

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/config/fully_async_ppo_megatron_trainer.yaml

@@ -0,0 +1,76 @@
+hydra:
+  searchpath:
+    - file://verl/trainer/config
+
+defaults:
+  - ppo_megatron_trainer
+  - _self_
+
+async_training:
+
+  # Maximum samples staleness threshold
+  staleness_threshold: 0.1
+
+  # Frequency of parameter synchronization between rollouter and trainer, 
+  # One step means trainer obtains a batch of required samples
+  trigger_parameter_sync_step: 4
+  
+  # The number of ppo_mini_batches that the FullyAsyncTrainer obtains once
+  require_batches: 1
+
+  # When synchronizing parameters, whether to interrupt rollouter and perform partial rollout
+  partial_rollout: True
+
+  # Whether to use rollout log probs for training
+  use_rollout_log_probs: True
+
+  # compute_prox_log_prob
+  compute_prox_log_prob: False
+
+  # whether to use trainer do_validate
+  use_trainer_do_validate: False
+
+
+  # checkpoint_engine config for accelerating parameter synchronization between rollouter and trainer
+  checkpoint_engine:
+    # Whether to use checkpoint_engine
+    enable: True
+
+    # Device buffer size for checkpoint_engine, default is 4096 MB
+    device_buffer_size_M: 4096
+
+    # Enable the pipeline for broadcasting and updating parameters, but it requires more device memory
+    overlap_broadcast_and_consume: False
+
+# Rollout config
+rollout:
+
+  # Number of nodes used in the rollout
+  nnodes: 1
+
+  # Number of GPUs per node                     
+  n_gpus_per_node: 8
+
+  # number of responses (i.e. num sample times). > 1 for grpo
+  n: 4
+
+  # total rollout samples # TODO rename to total_rollout_samples
+  total_rollout_steps: 100
+
+  # Number of epochs in training 
+  total_epochs: 10
+
+  # Test frequency, how many times a parameter update triggers a validation
+  test_freq: 1
+
+data:
+  # Number of samples generated, currently only support 1
+  gen_batch_size: 1
+
+actor_rollout_ref:
+  # checkpoint_engine config for accelerating parameter synchronization between rollouter and trainer
+  checkpoint_engine: ${oc.select:async_training.checkpoint_engine, null}
+
+  actor:
+    # Whether to use rollout log probs for training
+    use_rollout_log_probs: ${oc.select:async_training.use_rollout_log_probs, True}

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/config/fully_async_ppo_trainer.yaml

@@ -0,0 +1,76 @@
+hydra:
+  searchpath:
+    - file://verl/trainer/config
+
+defaults:
+  - ppo_trainer
+  - _self_
+
+async_training:
+
+  # Maximum samples staleness threshold
+  staleness_threshold: 0.1
+
+  # Frequency of parameter synchronization between rollouter and trainer, 
+  # One step means trainer obtains a batch of required samples
+  trigger_parameter_sync_step: 4
+  
+  # The number of ppo_mini_batches that the FullyAsyncTrainer obtains once
+  require_batches: 1
+
+  # When synchronizing parameters, whether to interrupt rollouter and perform partial rollout
+  partial_rollout: True
+
+  # Whether to use rollout log probs for training
+  use_rollout_log_probs: True
+
+  # compute_prox_log_prob
+  compute_prox_log_prob: False
+
+  # whether to use trainer do_validate
+  use_trainer_do_validate: False
+
+
+  # checkpoint_engine config for accelerating parameter synchronization between rollouter and trainer
+  checkpoint_engine:
+    # Whether to use checkpoint_engine
+    enable: True
+
+    # Device buffer size for checkpoint_engine, default is 4096 MB
+    device_buffer_size_M: 4096
+
+    # Enable the pipeline for broadcasting and updating parameters, but it requires more device memory
+    overlap_broadcast_and_consume: False
+
+# Rollout config
+rollout:
+
+  # Number of nodes used in the rollout
+  nnodes: 1
+
+  # Number of GPUs per node                     
+  n_gpus_per_node: 8
+
+  # number of responses (i.e. num sample times). > 1 for grpo
+  n: 4
+
+  # total rollout samples # TODO rename to total_rollout_samples
+  total_rollout_steps: 100
+
+  # Number of epochs in training 
+  total_epochs: 10
+
+  # Test frequency, how many times a parameter update triggers a validation
+  test_freq: 1
+
+data:
+  # Number of samples generated, currently only support 1
+  gen_batch_size: 1
+
+actor_rollout_ref:
+  # checkpoint_engine config for accelerating parameter synchronization between rollouter and trainer
+  checkpoint_engine: ${oc.select:async_training.checkpoint_engine, null}
+
+  actor:
+    # Whether to use rollout log probs for training
+    use_rollout_log_probs: ${oc.select:async_training.use_rollout_log_probs, True}

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/detach_utils.py

@@ -0,0 +1,363 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import numpy as np
+import torch
+
+from verl import DataProto
+from verl.experimental.agent_loop.agent_loop import AgentLoopOutput
+from verl.trainer.ppo.ray_trainer import compute_response_mask
+
+
+@dataclass
+class RolloutSample:
+    """Enhanced rollout sample containing both original batch info and AgentLoopOutput"""
+
+    # Original batch information
+    full_batch: Any
+
+    # AgentLoopOutput from generation
+    agent_loop_output_list: list[AgentLoopOutput]
+
+    # Metadata
+    sample_id: str
+    epoch: int
+
+    # Processing metadata
+    processing_times: list[float]
+    tool_calls: list[float]
+    param_version: int
+    param_version_start: list[int]
+    param_version_end: list[int]
+    rollout_status: dict[str, Any]
+
+
+@dataclass
+class ValidateMetrics:
+    """Metrics for validation"""
+
+    timing_raw: dict[str, Any]
+    metrics: Optional[dict[str, Any]] = None
+    global_steps: Optional[int] = None
+    param_version: Optional[int] = None
+
+
+def prepare_single_generation_data(batch_dict, config) -> DataProto:
+    """
+    Similar to the logic of ray_trainer._prepare_generate_batch, but for a single sample.
+    Separate the data used for generation from the original data.
+
+    Returns:
+        tuple: (original_batch_dict, gen_data_for_single_sample)
+    """
+
+    full_batch = DataProto.from_single_dict(batch_dict)
+
+    batch_keys_to_pop = []
+    non_tensor_batch_keys_to_pop = []
+
+    existing_batch_keys = [k for k in batch_keys_to_pop if k in full_batch.batch.keys()]
+    existing_non_tensor_keys = [k for k in non_tensor_batch_keys_to_pop if k in full_batch.non_tensor_batch.keys()]
+
+    if existing_batch_keys or existing_non_tensor_keys:
+        full_batch.pop(
+            batch_keys=existing_batch_keys,
+            non_tensor_batch_keys=existing_non_tensor_keys,
+        )
+
+    # Setting selected agent, that supports partial
+    if config.actor_rollout_ref.rollout.multi_turn.enable:
+        full_batch.non_tensor_batch["agent_name"] = np.array(
+            ["async_partial_tool_agent"] * len(full_batch), dtype=object
+        )
+    else:
+        full_batch.non_tensor_batch["agent_name"] = np.array(
+            ["partial_single_turn_agent"] * len(full_batch), dtype=object
+        )
+
+    # Add global step count to generated data
+    full_batch = full_batch.repeat(repeat_times=config.actor_rollout_ref.rollout.n, interleave=True)
+    return full_batch
+
+
+def assemble_batch_from_rollout_samples(
+    rollout_samples: list[RolloutSample], tokenizer, config, balance_batch=None
+) -> DataProto:
+    """
+    Assemble gen_batch_output from RolloutSample objects
+    Assembles batches from RolloutSample objects, similar to the _post_generate_batch logic in ray_trainer.
+
+    Args:
+        rollout_samples: List of RolloutSample objects
+        tokenizer: Tokenizer instance
+        config: Configuration object containing trainer settings
+        balance_batch: Whether to balance the batch (simplified version)
+
+    Returns:
+        DataProto: Assembled gen_batch_output
+
+    Raises:
+        ValueError: If rollout_samples is empty
+    """
+    start_time = time.time()
+
+    if not rollout_samples:
+        raise ValueError("Empty rollout_samples provided for batch assembly")
+
+    print(f"[BatchUtils] Assembling batch from {len(rollout_samples)} RolloutSample objects")
+
+    rollout_samples_batch = []
+    processing_times = []
+    tool_calls = []
+    rollout_status = rollout_samples[0].rollout_status
+    # Add a prefix to all rollout_status keys
+    rollout_status = {f"fully_async/{key}": value for key, value in rollout_status.items()}
+
+    for rs in rollout_samples:
+        rollout_samples_batch.append(rs.full_batch)
+    final_batch = DataProto.concat(rollout_samples_batch)
+
+    # Calculate response_mask (if not present)
+    if "response_mask" not in final_batch.batch.keys():
+        final_batch.batch["response_mask"] = compute_response_mask(final_batch)
+
+    if balance_batch:
+        balance_batch(final_batch, metrics={})
+
+    # Calculate the global valid token number
+    if "attention_mask" in final_batch.batch:
+        final_batch.meta_info["global_token_num"] = torch.sum(final_batch.batch["attention_mask"], dim=-1).tolist()
+
+    processing_times = final_batch.non_tensor_batch["processing_times"]
+    tool_calls = final_batch.non_tensor_batch["tool_calls_times"]
+    # Collect statistics
+
+    processing_time_stats = {
+        "processing_time/avg": np.mean(processing_times),
+        "processing_time/max": np.max(processing_times),
+        "processing_time/min": np.min(processing_times),
+        "processing_time/tp50": np.percentile(processing_times, 50),
+        "processing_time/tp99": np.percentile(processing_times, 99),
+        "processing_time/tp95": np.percentile(processing_times, 95),
+    }
+    tool_calls_stats = {}
+    if len(tool_calls) > 0:
+        tool_calls_stats = {
+            "timing_s/agent_loop/tool_calls/max": np.max(tool_calls),
+            "timing_s/agent_loop/tool_calls/min": np.min(tool_calls),
+            "timing_s/agent_loop/tool_calls/mean": np.mean(tool_calls),
+        }
+    processing_time_stats = {f"fully_async/{key}": value for key, value in processing_time_stats.items()}
+
+    param_version_start = final_batch.non_tensor_batch["param_version_start"]
+    param_version_end = final_batch.non_tensor_batch["param_version_end"]
+    param_version_diff = [abs(a - b) for a, b in zip(param_version_end, param_version_start, strict=False)]
+    num_diff0 = param_version_diff.count(0)
+    partial_stats = {
+        "fully_async/partial/total_partial_num": len(param_version_diff) - num_diff0,
+        "fully_async/partial/partial_ratio": (len(param_version_diff) - num_diff0) / len(param_version_diff),
+        "fully_async/partial/max_partial_span": max(param_version_diff),
+    }
+    # add meta_info
+    param_versions = [rs.param_version for rs in rollout_samples]
+    trajectorys_param_versions = final_batch.non_tensor_batch["param_version_end"]
+
+    final_batch.meta_info.update(
+        {
+            "rollout_param_versions": param_versions,
+            "param_version_diversity": len(set(param_versions)) if param_versions else 0,
+            "trajectory_param_versions": trajectorys_param_versions,
+            **processing_time_stats,
+            **rollout_status,
+            **partial_stats,
+            **tool_calls_stats,
+        }
+    )
+
+    print(f"[BatchUtils] Batch assembly completed in {time.time() - start_time:.2f}s")
+
+    return final_batch
+
+
+class MetricsAggregator:
+    """Metrics aggregator, used to combine metrics from multiple training steps"""
+
+    def __init__(self, total_gpus: int):
+        # Store all values ​​for each metric
+        self.metric_values: dict[str, list[float]] = defaultdict(list)
+        # Store the number of samples at each step for weighted averaging
+        self.sample_counts: list[int] = []
+        # Store the timestamp of each step for time-related calculations
+        self.timestamps: list[float] = []
+        # Step Count
+        self.step_count = 0
+        # total num gpus used
+        self.total_gpus = total_gpus
+
+        # Metric aggregation rule configuration
+        self.aggregation_rules = self._init_aggregation_rules()
+
+    def _init_aggregation_rules(self) -> dict[str, dict[str, list[str]]]:
+        """Initialize metrics aggregation rules"""
+        return {
+            # Time-Based metrics, can add metrics here
+            "time_sum": ["perf/time_per_step"],
+            "min": ["timing_s/agent_loop/tool_calls/min"],
+            "avg": ["timing_s/agent_loop/tool_calls/mean"],
+            "max": ["timing_s/agent_loop/tool_calls/max"],
+            "last": [
+                "fully_async/count/total_generated_samples",
+                "fully_async/count/stale_samples_processed",
+                "fully_async/count/stale_trajectory_processed",
+                "fully_async/count/current_param_version",
+                "fully_async/count/dropped_stale_samples",
+                "training/global_step",  # TODO change name to: total_step
+            ],
+        }
+
+    def add_step_metrics(self, metrics: dict[str, Any], sample_count: int, timestamp: float = None):
+        """Adding a single-step metrics"""
+        if timestamp is None:
+            timestamp = time.time()
+
+        self.sample_counts.append(sample_count)
+        self.timestamps.append(timestamp)
+        self.step_count += 1
+
+        # Store all metrics values
+        for key, value in metrics.items():
+            if isinstance(value, int | float | np.number):
+                self.metric_values[key].append(float(value))
+            elif isinstance(value, torch.Tensor):
+                self.metric_values[key].append(float(value.item()))
+
+    def _get_aggregation_type(self, metric_name: str) -> str:
+        """Determine the aggregation type based on the metric name"""
+        for agg_type, metric_list in self.aggregation_rules.items():
+            if metric_name in metric_list:
+                return agg_type
+
+        metric_lower = metric_name.lower()
+        if any(keyword in metric_lower for keyword in ["timing_s/"]):
+            return "time_sum"
+        if any(keyword in metric_lower for keyword in ["mean", "avg", "average"]):
+            return "avg"
+        if any(keyword in metric_lower for keyword in ["max", "maximum"]):
+            return "max"
+        if any(keyword in metric_lower for keyword in ["min", "minimum"]):
+            return "min"
+        if any(keyword in metric_lower for keyword in ["sum", "total"]):
+            return "sum"
+        if any(keyword in metric_lower for keyword in ["weighted_avg"]):
+            return "weighted_avg"
+
+        return "avg"
+
+    def _aggregate_single_metric(self, metric_name: str, values: list[float]) -> float:
+        """Aggregating a single metric"""
+        if not values:
+            return 0.0
+
+        agg_type = self._get_aggregation_type(metric_name)
+
+        if agg_type == "last":
+            return values[-1]
+
+        elif agg_type == "weighted_avg":
+            # Weighted average
+            if len(values) != len(self.sample_counts):
+                # If the lengths do not match, use a simple average
+                return sum(values) / len(values)
+
+            total_samples = sum(self.sample_counts)
+            if total_samples == 0:
+                return sum(values) / len(values)
+
+            weighted_sum = sum(v * c for v, c in zip(values, self.sample_counts, strict=False))
+            return weighted_sum / total_samples
+
+        elif agg_type == "sum" or agg_type == "time_sum":
+            return sum(values)
+
+        elif agg_type == "avg":
+            return sum(values) / len(values)
+
+        elif agg_type == "max":
+            return max(values)
+
+        elif agg_type == "min":
+            return min(values)
+
+        else:
+            # Default average
+            return sum(values) / len(values)
+
+    def get_aggregated_metrics(self) -> dict[str, Any]:
+        """aggregated metrics"""
+        t = time.time()
+        if self.step_count == 0:
+            return {}
+
+        aggregated = {}
+
+        # Aggregate all metrics
+        for metric_name, values in self.metric_values.items():
+            aggregated[metric_name] = self._aggregate_single_metric(metric_name, values)
+
+        # Aggregate special metrics
+        aggregated = self._special_metrics_aggergate(aggregated)
+
+        print(f"aggregated metrics done. cost {time.time() - t}")
+
+        return aggregated
+
+    def _special_metrics_aggergate(self, aggregated: dict[str, Any]) -> dict[str, Any]:
+        """calculate special metrics"""
+
+        # global_seqlen/minmax_diff
+        if "global_seqlen/minmax_diff" in aggregated.keys():
+            aggregated["global_seqlen/minmax_diff"] = aggregated["global_seqlen/max"] - aggregated["global_seqlen/min"]
+
+        # perf/throughput
+        REQUIRED_PERF_KEYS = {"perf/throughput", "perf/total_num_tokens", "perf/time_per_step"}
+        if REQUIRED_PERF_KEYS.issubset(aggregated):
+            aggregated["perf/throughput"] = aggregated["perf/total_num_tokens"] / (
+                aggregated["perf/time_per_step"] * self.total_gpus
+            )
+
+        # trainer/idle_ratio
+        if "timing_s/gen" in aggregated.keys() and "timing_s/step" in aggregated.keys():
+            aggregated["trainer/idle_ratio"] = aggregated["timing_s/gen"] / aggregated["timing_s/step"]
+
+        return aggregated
+
+    def reset(self):
+        """Reset Aggregator"""
+        self.metric_values.clear()
+        self.sample_counts.clear()
+        self.timestamps.clear()
+        self.step_count = 0
+
+    def get_current_stats(self) -> dict[str, Any]:
+        """Get statistics about the current aggregation state (for debugging)"""
+        return {
+            "step_count": self.step_count,
+            "metric_count": len(self.metric_values),
+            "total_samples": sum(self.sample_counts),
+            "metric_names": list(self.metric_values.keys()),
+        }

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/fsdp2_utils.py

@@ -0,0 +1,125 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+import torch.distributed as dist
+from packaging import version
+from torch.distributed.tensor import DTensor
+from torch.distributed.tensor._dtensor_spec import DTensorSpec
+
+if version.parse(torch.__version__) < version.parse("2.6"):
+    raise RuntimeError("PyTorch 2.6 or higher is required to use fstp_utils.")
+
+
+def fsdp2_sharded_save_to_cpu(
+    model: torch.nn.Module,
+) -> tuple[dict[str, tuple[torch.Tensor, DTensorSpec]], DTensorSpec]:
+    """
+    Sharded Save: Each process only saves the local DTensor shard from its own GPU to CPU memory.
+
+    Args:
+        model: FSDP2-wrapped model whose parameters are of DTensor type.
+
+    Returns:
+        cpu_sharded_state: Dictionary of CPU shards for the current process.
+                          Key = parameter name, Value = (CPU shard tensor, original DTensorSpec)
+        global_spec: DTensorSpec of the first parameter (used to verify global rules during loading)
+    """
+    cpu_sharded_state = {}
+    global_spec = None  # Record global sharding rules (all parameters follow the same spec)
+
+    for param_name, param in model.named_parameters():
+        # Only process sharded parameters of DTensor type (core parameters of FSDP2)
+        if not isinstance(param, DTensor):
+            # Save non-sharded parameters (e.g., running_mean of BatchNorm) as local data
+            cpu_tensor = param.detach().cpu()
+            cpu_sharded_state[param_name] = (cpu_tensor, None)
+            continue
+
+        # Record global sharding rules (take spec of the first DTensor to ensure consistency)
+        if global_spec is None:
+            global_spec = param._spec
+            assert hasattr(global_spec, "device_mesh"), "DTensorSpec must contain 'device_mesh' attribute"
+            assert hasattr(global_spec, "placements"), "DTensorSpec must contain 'placements' attribute"
+
+        # 1. Extract local shard data from the current GPU (_local_tensor)
+        local_gpu_tensor = param._local_tensor  # Local shard attribute defined in your DTensor class
+        # 2. Move to CPU memory and detach from computation graph
+        local_cpu_tensor = local_gpu_tensor.detach().cpu()
+        # 3. Save CPU shard + original DTensorSpec (ensure sharding rules remain unchanged)
+        cpu_sharded_state[param_name] = (local_cpu_tensor, param._spec)
+
+    assert global_spec is not None, "No DTensor-type parameters found in the model. FSDP2 sharding may not be enabled."
+    return cpu_sharded_state, global_spec
+
+
+def fsdp2_sharded_load_from_cpu(
+    model: torch.nn.Module,
+    cpu_sharded_state: dict[str, tuple[torch.Tensor, Optional[DTensorSpec]]],
+    target_spec: DTensorSpec,
+) -> None:
+    """
+    Sharded Load: Each process only loads the CPU shard it is responsible for to the GPU,
+                  keeping sharding rules unchanged.
+
+    Args:
+        model: FSDP2 model to be restored (must have the same structure as when saved)
+        cpu_sharded_state: Shard data read from CPU memory by the current process
+                          (from fsdp2_sharded_save_to_cpu)
+        target_spec: Global DTensorSpec from saving (used to verify sharding rule consistency)
+    """
+    # Verify device_mesh consistency (core: ensure loaded shards map to original GPUs)
+    current_device_mesh = None
+    for param in model.parameters():
+        if isinstance(param, DTensor):
+            current_device_mesh = param._spec.device_mesh
+            break
+    assert current_device_mesh is not None, "DTensor parameters not initialized in the model to be loaded"
+    assert current_device_mesh == target_spec.device_mesh, (
+        f"device_mesh mismatch during loading! Original: {target_spec.device_mesh}, Current: {current_device_mesh}"
+    )
+
+    for param_name, param in model.named_parameters():
+        # Skip parameters not in the saved state (e.g., newly added parameters)
+        if param_name not in cpu_sharded_state:
+            continue
+
+        # Extract CPU shard data and original Spec
+        local_cpu_tensor, saved_spec = cpu_sharded_state[param_name]
+
+        # Handle different parameter types: DTensor sharded parameters vs. regular parameters
+        if isinstance(param, DTensor):
+            # 1. Verify sharding rule consistency (placements must match original Spec)
+            assert saved_spec is not None, f"DTensorSpec missing in saved state for parameter {param_name}"
+            assert saved_spec.placements == target_spec.placements, (
+                f"Sharding strategy mismatch for parameter {param_name} (conflicts with global rules)!"
+            )
+
+            # 2. Move CPU shard data to the current GPU (device of param._local_tensor)
+            target_device = param._local_tensor.device
+            local_gpu_tensor = local_cpu_tensor.to(target_device)
+
+            # 3. Restore to DTensor's local shard (directly copy to _local_tensor, keep spec unchanged)
+            param._local_tensor.copy_(local_gpu_tensor)
+
+        else:
+            # Regular parameters: load directly to original device
+            target_device = param.device
+            param.data.copy_(local_cpu_tensor.to(target_device))
+
+    # Process synchronization: ensure all processes complete loading before proceeding
+    dist.barrier()

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/fsdp_workers.py

@@ -0,0 +1,247 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import time
+
+import torch
+import torch.distributed
+from omegaconf import DictConfig
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+
+from verl.experimental.fully_async_policy.base_detach_sync import BaseDetachNcclSync
+from verl.experimental.fully_async_policy.fsdp2_utils import fsdp2_sharded_load_from_cpu, fsdp2_sharded_save_to_cpu
+from verl.single_controller.base.decorator import Dispatch, register
+from verl.utils.device import (
+    get_device_name,
+    get_torch_device,
+)
+from verl.utils.fsdp_utils import (
+    fsdp_version,
+    load_fsdp_model_to_gpu,
+    offload_fsdp_model_to_cpu,
+)
+from verl.workers.fsdp_workers import AsyncActorRolloutRefWorker, CriticWorker
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+device_name = get_device_name()
+
+__all__ = ["DetachActorWorker", "DetachAsyncRolloutWorker", "CriticWorker"]
+
+
+class DetachNcclSync(BaseDetachNcclSync, AsyncActorRolloutRefWorker):
+    def __init__(self, config: DictConfig, role: str):
+        BaseDetachNcclSync.__init__(self, config, role)
+        AsyncActorRolloutRefWorker.__init__(self, config, role)
+
+    def _get_actor_params(self):
+        pass
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False)
+    def sync_rollout_weights(self, sync_group_name="actor_rollout"):
+        assert (self._is_actor or self._is_rollout) and not self.config.hybrid_engine
+        assert hasattr(self, "_weights_info") and self._weights_info is not None
+
+        if self._is_actor and self._is_offload_param:
+            load_fsdp_model_to_gpu(self.actor_module_fsdp)
+        params = self._get_actor_params() if self._is_actor else None
+        rollout_name = self.config.rollout.name
+
+        inference_model = None
+        if self._is_rollout:
+            if rollout_name == "vllm":
+                inference_model = BaseDetachNcclSync.get_inference_model(self.rollout)
+
+                from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader
+
+                patch_vllm_moe_model_weight_loader(inference_model)
+            elif rollout_name == "sglang":
+                inference_model = self.rollout._engine
+                # For ServerAdapter, _engine might be None and needs async initialization
+                if inference_model is None:
+                    # Initialize the server adapter engine
+                    print("[sync_rollout_weights] Initialize server adapter engine")
+
+                    async def init_engine():
+                        if hasattr(self.rollout, "_init_server_adapter"):
+                            await self.rollout._init_server_adapter()
+                        else:
+                            print("[sync_rollout_weights] No _init_server_adapter method found")
+                        return self.rollout._engine
+
+                    inference_model = self._run_async_safely(init_engine())
+                    if inference_model is None:
+                        raise RuntimeError(
+                            f"Failed to initialize rollout engine. "
+                            f"rollout type: {type(self.rollout)}, "
+                            f"has _init_server_adapter: {hasattr(self.rollout, '_init_server_adapter')}"
+                        )
+            else:
+                raise NotImplementedError(f"Unknown rollout name: {rollout_name}")
+
+        if rollout_name == "sglang" and self._is_rollout:
+            self._sync_sglang_weights(inference_model, params, sync_group_name)
+        else:
+            self._sync_vllm_weights(inference_model, params, sync_group_name)
+
+        if self._is_actor and self._is_offload_param:
+            offload_fsdp_model_to_cpu(self.actor_module_fsdp)
+        get_torch_device().empty_cache()
+
+    def cache_actor_weights_to_cpu(self):
+        self.cpu_named_params = {}
+        if self._is_actor:
+            params = self._get_actor_params()
+            local_rank = torch.distributed.get_rank()
+            world_size = torch.distributed.get_world_size()
+
+            for tensor_idx, (key, _, _) in enumerate(self._weights_info):
+                origin_data = params[key]
+                if hasattr(origin_data, "full_tensor"):
+                    origin_data = origin_data.full_tensor()
+
+                if tensor_idx % world_size == local_rank:
+                    self.cpu_named_params[key] = origin_data.to("cpu", non_blocking=True)
+            get_torch_device().synchronize()
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False)
+    def sync_rollout_weights_by_checkpoint(self, sync_group_name="actor_rollout"):
+        assert (self._is_actor or self._is_rollout) and not self.config.hybrid_engine
+        assert hasattr(self, "_weights_info") and self._weights_info is not None
+
+        # Load model to GPU
+        load_start_time = time.time()
+        if self._is_actor and self._is_offload_param:
+            load_fsdp_model_to_gpu(self.actor_module_fsdp)
+        load_duration = time.time() - load_start_time
+
+        from ray.util.collective import collective
+
+        # Cache actor weights to CPU and measure the time taken
+        cache_start_time = time.time()
+        self.cache_actor_weights_to_cpu()
+        cache_end_time = time.time()
+        cache_duration = cache_end_time - cache_start_time
+
+        # Register the cached weights into the checkpoint engine
+        self.checkpoint_engine.register_checkpoint(self._weights_info, self.cpu_named_params)
+        register_end_time = time.time()
+        register_duration = register_end_time - cache_end_time
+        self.cpu_named_params = {}
+
+        collective.barrier(group_name=sync_group_name)
+        update_start_time = time.time()
+
+        inference_model = None
+        if self._is_rollout:
+            inference_model = BaseDetachNcclSync.get_inference_model(self.rollout)
+            from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader
+
+            patch_vllm_moe_model_weight_loader(inference_model)
+
+        # Update the checkpoint with the inference model and broadcast weights
+        self.checkpoint_engine.update_checkpoint(
+            inference_model=inference_model,
+            group_name=sync_group_name,
+            overlap_broadcast_and_consume=self.config.checkpoint_engine.overlap_broadcast_and_consume,
+        )
+
+        update_end_time = time.time()
+        update_duration = update_end_time - update_start_time
+
+        offload_start_time = time.time()
+        if self._is_actor and self._is_offload_param:
+            offload_fsdp_model_to_cpu(self.actor_module_fsdp)
+        offload_duration = time.time() - offload_start_time
+
+        print(
+            f"sync_rollout_weights_by_checkpoint finish!, rank:{torch.distributed.get_rank()},"
+            f" is_actor:{self._is_actor}, is_rollout:{self._is_rollout},"
+            f" total cost:{update_end_time - cache_start_time} seconds, while cache cost {cache_duration} seconds, "
+            f" register cost {register_duration} seconds, update cost {update_duration} seconds"
+        )
+
+        if self._is_actor and self._is_offload_param:
+            print(
+                f"sync_rollout_weights_by_checkpoint load model to gpu cost {load_duration} seconds,"
+                f" offload model to cpu cost {offload_duration} seconds"
+            )
+
+
+class DetachActorWorker(DetachNcclSync):
+    def __init__(self, config: DictConfig, role: str):
+        print("[DetachAsyncRolloutWorker] Initializing via DetachNcclSync...")
+        DetachNcclSync.__init__(self, config, role)
+
+    def _get_actor_params(self):
+        assert self._is_actor
+        params = self.actor_module_fsdp.state_dict()
+        from verl.utils.model import convert_weight_keys
+
+        params = convert_weight_keys(
+            params, getattr(self.actor_module_fsdp, "_fsdp_wrapped_module", self.actor_module_fsdp)
+        )
+        return params
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def get_actor_weights_info(self):
+        assert self._is_actor
+        if hasattr(self, "_weights_info"):
+            return self._weights_info
+        if fsdp_version(self.actor_module_fsdp) == 1:
+            from torch.distributed.fsdp.api import ShardedStateDictConfig, StateDictType
+
+            FSDP.set_state_dict_type(
+                self.actor_module_fsdp,
+                state_dict_type=StateDictType.SHARDED_STATE_DICT,
+                state_dict_config=ShardedStateDictConfig(),
+            )
+        params = self._get_actor_params()
+        ret = []
+        for key, tensor in params.items():
+            ret.append((key, tensor.size(), tensor.dtype))
+        self._weights_info = ret
+        return ret
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def save_model_to_cpu(self, n):
+        if not hasattr(self, "cpu_saved_models"):
+            self.cpu_saved_models = {}
+        self.cpu_saved_models[n] = fsdp2_sharded_save_to_cpu(self.actor_module_fsdp)
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def restore_model_from_cpu(self, n):
+        if n in self.cpu_saved_models:
+            cpu_sharded_state, global_spec = self.cpu_saved_models[n]
+            fsdp2_sharded_load_from_cpu(self.actor_module_fsdp, cpu_sharded_state, global_spec)
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def clear_cpu_model(self, n):
+        if n in self.cpu_saved_models:
+            del self.cpu_saved_models[n]
+
+
+class DetachAsyncRolloutWorker(DetachNcclSync):
+    def __init__(self, config: DictConfig, role: str):
+        print(f"[DetachAsyncRolloutWorker] {DetachAsyncRolloutWorker.__mro__}")
+        DetachNcclSync.__init__(self, config, role)
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def set_actor_weights_info(self, weights_info):
+        assert self._is_rollout
+        self._weights_info = weights_info

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/fully_async_main.py

@@ -0,0 +1,312 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import os
+import socket
+import threading
+from pprint import pprint
+
+import hydra
+import ray
+from omegaconf import OmegaConf
+
+from verl.experimental.fully_async_policy.fully_async_rollouter import FullyAsyncRollouter
+from verl.experimental.fully_async_policy.fully_async_trainer import FullyAsyncTrainer
+from verl.experimental.fully_async_policy.message_queue import MessageQueue, MessageQueueClient
+from verl.trainer.ppo.ray_trainer import ResourcePoolManager
+from verl.trainer.ppo.utils import Role, need_reference_policy
+from verl.utils.fs import copy_to_local
+
+
+def create_resource_pool_manager(config, roles: list) -> ResourcePoolManager:
+    """
+    Create resource pool manager
+
+    Args:
+        config: Configuration object
+        roles: List of roles that need to create resource pools
+
+    Returns:
+        ResourcePoolManager: Resource pool manager
+    """
+    resource_pool_spec = {}
+    mapping = {}
+
+    # Actor/Critic resource pool
+    if any(role in roles for role in [Role.Actor, Role.ActorRollout, Role.Critic, Role.RefPolicy, Role.RewardModel]):
+        assert config.trainer.n_gpus_per_node > 0, "config.trainer.n_gpus_per_node must be greater than 0"
+        assert config.trainer.nnodes > 0, "config.trainer.nnodes must be greater than 0"
+
+        trainer_pool = [config.trainer.n_gpus_per_node] * config.trainer.nnodes
+        resource_pool_spec["trainer_pool"] = trainer_pool
+
+        # Map training-related roles to the same resource pool
+        for role in [Role.Actor, Role.ActorRollout, Role.Critic, Role.RefPolicy, Role.RewardModel]:
+            if role in roles:
+                mapping[role] = "trainer_pool"
+
+    # Rollout resource pool
+    if Role.Rollout in roles:
+        assert config.rollout.n_gpus_per_node > 0, "config.rollout.n_gpus_per_node must be greater than 0"
+        assert config.rollout.nnodes > 0, "config.rollout.nnodes must be greater than 0"
+
+        rollout_pool = [config.rollout.n_gpus_per_node] * config.rollout.nnodes
+        resource_pool_spec["rollout_pool"] = rollout_pool
+        mapping[Role.Rollout] = "rollout_pool"
+
+    return ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping)
+
+
+def create_role_worker_mapping(config):
+    """
+    Create mapping from roles to worker classes
+
+    Args:
+        config: Configuration object
+
+    Returns:
+        dict: Mapping from roles to worker classes
+    """
+    # Select worker class based on strategy
+    if config.actor_rollout_ref.actor.strategy in ["fsdp", "fsdp2"]:
+        assert config.actor_rollout_ref.actor.strategy == config.critic.strategy
+        from verl.experimental.fully_async_policy.fsdp_workers import (
+            CriticWorker,
+            DetachActorWorker,
+            DetachAsyncRolloutWorker,
+        )
+        from verl.single_controller.ray import RayWorkerGroup
+
+        ray_worker_group_cls = RayWorkerGroup
+
+    elif config.actor_rollout_ref.actor.strategy == "megatron":
+        assert config.critic.strategy == "megatron"
+        from verl.experimental.fully_async_policy.megatron_worker import (
+            CriticWorker,
+            DetachActorWorker,
+            DetachAsyncRolloutWorker,
+        )
+        from verl.single_controller.ray import RayWorkerGroup
+
+        ray_worker_group_cls = RayWorkerGroup
+    else:
+        raise NotImplementedError(f"Unsupported strategy: {config.actor_rollout_ref.actor.strategy}")
+
+    train_role = Role.ActorRollout if config.async_training.use_trainer_do_validate else Role.Actor
+    role_worker_mapping = {
+        train_role: ray.remote(DetachActorWorker),
+        Role.Rollout: ray.remote(DetachAsyncRolloutWorker),
+        Role.Critic: ray.remote(CriticWorker),
+    }
+
+    if config.reward_model.enable:
+        if config.reward_model.strategy in ["fsdp", "fsdp2"]:
+            from verl.workers.fsdp_workers import RewardModelWorker
+        elif config.reward_model.strategy == "megatron":
+            from verl.workers.megatron_workers import RewardModelWorker
+        else:
+            raise NotImplementedError
+
+        role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker)
+
+    # Add reference policy (if KL loss or reward is required)
+    if need_reference_policy(config):
+        role_worker_mapping[Role.RefPolicy] = ray.remote(DetachActorWorker)
+
+    return role_worker_mapping, ray_worker_group_cls
+
+
+@ray.remote(num_cpus=1)
+class FullyAsyncTaskRunner:
+    """
+    Ray remote class for executing distributed PPO training tasks.
+    """
+
+    def __init__(self):
+        self.running = False
+        self.components = {}
+        self.shutdown_event = threading.Event()
+
+    def run(self, config):
+        print("[ASYNC MAIN] Starting fully async PPO training...")
+        self._initialize_components(config)
+        self._run_training_loop()
+
+    def _initialize_components(self, config) -> None:
+        print(f"[ASYNC MAIN] TaskRunner hostname: {socket.gethostname()}, PID: {os.getpid()}")
+        pprint(OmegaConf.to_container(config, resolve=True))
+        OmegaConf.resolve(config)
+
+        print("[ASYNC MAIN] Initializing model and tokenizer...")
+        local_path = copy_to_local(
+            config.actor_rollout_ref.model.path, use_shm=config.actor_rollout_ref.model.get("use_shm", False)
+        )
+        from verl.utils import hf_processor, hf_tokenizer
+
+        trust_remote_code = config.data.get("trust_remote_code", False)
+        tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code)
+
+        # Used for multimodal LLM, could be None
+        processor = hf_processor(local_path, trust_remote_code=trust_remote_code, use_fast=True)
+
+        self.components["tokenizer"] = tokenizer
+        self.components["processor"] = processor
+        self.components["config"] = config
+
+        print("[ASYNC MAIN] Creating worker mapping and resource pools...")
+        role_worker_mapping, ray_worker_group_cls = create_role_worker_mapping(config)
+        self.components["role_worker_mapping"] = role_worker_mapping
+        self.components["ray_worker_group_cls"] = ray_worker_group_cls
+
+        print("[ASYNC MAIN] Creating FullyAsyncRollouter...")
+        self._create_rollouter(config)
+
+        print("[ASYNC MAIN] Creating FullyAsyncTrainer...")
+        self._create_trainer(config)
+
+        # sync total_train_steps between rollouter and trainer
+        total_train_steps = ray.get(self.components["rollouter"].get_total_train_steps.remote())
+        print(f"total_train_steps {total_train_steps}")
+        ray.get(self.components["trainer"].set_total_train_steps.remote(total_train_steps))
+
+        # max_queue_size
+        max_queue_size = ray.get(self.components["rollouter"].get_max_queue_size.remote())
+        print(f"[ASYNC MAIN] Creating MessageQueue... max_queue_size {max_queue_size}")
+        message_queue = MessageQueue.remote(config, max_queue_size)
+        message_queue_client = MessageQueueClient(message_queue)
+        self.components["message_queue"] = message_queue
+        self.components["message_queue_client"] = message_queue_client
+
+        ray.get(self.components["rollouter"].set_message_queue_client.remote(self.components["message_queue_client"]))
+        ray.get(self.components["trainer"].set_message_queue_client.remote(self.components["message_queue_client"]))
+
+        print("[ASYNC MAIN] Setting up parameter synchronization...")
+        from verl.experimental.fully_async_policy.param_sync import ParameterSynchronizer
+
+        param_synchronizer = ParameterSynchronizer.remote(
+            config=config,
+            trainer=self.components["trainer"],
+            rollouter=self.components["rollouter"],
+            mq=self.components["message_queue_client"],
+        )
+        ray.get(self.components["trainer"].set_parameter_synchronizer.remote(param_synchronizer))
+
+        # load checkpoint and sync parameter before doing anything
+        val_before_train = config.trainer.get("val_before_train", True)
+        # param_version resume from ckpt or default 0
+        param_version = ray.get(self.components["trainer"].load_checkpoint.remote())
+        ray.get(self.components["rollouter"].load_checkpoint.remote())
+        ray.get(
+            param_synchronizer.sync_weights.remote(
+                version=param_version,
+                validate=val_before_train,
+                use_trainer_do_validate=config.async_training.use_trainer_do_validate,
+            )
+        )
+        ray.get(param_synchronizer.wait_last_valid.remote())
+
+        self.components["param_synchronizer"] = param_synchronizer
+        print("[ASYNC MAIN] All components initialized successfully")
+
+    def _create_rollouter(self, config) -> None:
+        rollouter = FullyAsyncRollouter.remote(
+            config=config,
+            tokenizer=self.components["tokenizer"],
+            role_worker_mapping={Role.Rollout: self.components["role_worker_mapping"][Role.Rollout]},
+            resource_pool_manager=create_resource_pool_manager(config, roles=[Role.Rollout]),
+            ray_worker_group_cls=self.components["ray_worker_group_cls"],
+            processor=self.components["processor"],
+            device_name=config.trainer.device,
+        )
+
+        ray.get(rollouter.init_workers.remote())
+        ray.get(rollouter.set_max_required_samples.remote())
+
+        self.components["rollouter"] = rollouter
+        print("[ASYNC MAIN] Rollouter created and initialized successfully")
+
+    def _create_trainer(self, config) -> None:
+        trainer_role_mapping = {
+            role: worker_cls
+            for role, worker_cls in self.components["role_worker_mapping"].items()
+            if role != Role.Rollout
+        }
+
+        trainer = FullyAsyncTrainer.remote(
+            config=config,
+            tokenizer=self.components["tokenizer"],
+            role_worker_mapping=trainer_role_mapping,
+            resource_pool_manager=create_resource_pool_manager(config, roles=list(trainer_role_mapping.keys())),
+            ray_worker_group_cls=self.components["ray_worker_group_cls"],
+            processor=self.components["processor"],
+            device_name=config.trainer.device,
+        )
+
+        ray.get(trainer.init_workers.remote())
+        self.components["trainer"] = trainer
+        print("[ASYNC MAIN] FullyAsyncTrainer created and initialized successfully")
+
+    def _run_training_loop(self):
+        self.running = True
+
+        print("[ASYNC MAIN] Starting Rollouter and Trainer...")
+        rollouter_future = self.components["rollouter"].fit.remote()
+        trainer_future = self.components["trainer"].fit.remote()
+
+        futures = [rollouter_future, trainer_future]
+
+        try:
+            while futures:
+                # Use ray.wait to monitor all futures and return when any one is completed.
+                done_futures, remaining_futures = ray.wait(futures, num_returns=1, timeout=None)
+
+                for future in done_futures:
+                    try:
+                        ray.get(future)
+                        print("[ASYNC MAIN] One component completed successfully")
+                    except Exception as e:
+                        print(f"[ASYNC MAIN] Component failed with error: {e}")
+                        for remaining_future in remaining_futures:
+                            ray.cancel(remaining_future)
+                        raise e
+
+                futures = remaining_futures
+
+        except Exception as e:
+            print(f"[ASYNC MAIN] Training failed: {e}")
+            for future in futures:
+                ray.cancel(future)
+            raise
+        finally:
+            asyncio.run(self.components["message_queue_client"].clear_queue())
+            print("[ASYNC MAIN] Training completed or interrupted")
+
+
+@hydra.main(config_path="config", config_name="fully_async_ppo_trainer", version_base=None)
+def main(config):
+    from verl.trainer.main_ppo import run_ppo
+
+    # Ensure async training config exists
+    if not hasattr(config, "async_training"):
+        raise RuntimeError("must set async_training config")
+    from time import time
+
+    start_time = time()
+    run_ppo(config, task_runner_class=FullyAsyncTaskRunner)
+    print(f"total time: {time() - start_time:.2f} seconds")
+
+
+if __name__ == "__main__":
+    main()

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/fully_async_rollouter.py

@@ -0,0 +1,793 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import functools
+import multiprocessing
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pprint import pformat
+
+import numpy as np
+import ray
+import torch
+from ray import ObjectRef
+
+from verl.experimental.fully_async_policy.detach_utils import (
+    RolloutSample,
+    ValidateMetrics,
+    prepare_single_generation_data,
+)
+from verl.experimental.fully_async_policy.message_queue import MessageQueueClient
+from verl.experimental.fully_async_policy.ray_trainer import FullyAsyncRayPPOTrainer
+from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup
+from verl.trainer.ppo.ray_trainer import ResourcePoolManager
+from verl.trainer.ppo.reward import load_reward_manager
+from verl.trainer.ppo.utils import Role, WorkerType
+from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path
+from verl.utils.profiler import marked_timer
+from verl.utils.tracking import ValidationGenerationsLogger
+
+
+@ray.remote(num_cpus=10, max_concurrency=100)
+class FullyAsyncRollouter(FullyAsyncRayPPOTrainer):
+    """
+    Asynchronous sample generator, responsible for continuously generating training samples
+    and putting them into MessageQueue
+    Based on the mature implementation improvements of OneStepOffRayTrainer
+    """
+
+    def __init__(
+        self,
+        config,
+        tokenizer,
+        role_worker_mapping: dict[Role, WorkerType],
+        resource_pool_manager: ResourcePoolManager,
+        ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup,
+        processor=None,
+        reward_fn=None,
+        val_reward_fn=None,
+        device_name=None,
+    ):
+        # Store the tokenizer for text processing
+        self.tokenizer = tokenizer
+        self.processor = processor
+        self.config = config
+        self.reward_fn = load_reward_manager(
+            config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {})
+        )
+        self.val_reward_fn = load_reward_manager(
+            config, tokenizer, num_examine=1, **config.reward_model.get("reward_kwargs", {})
+        )
+        self.hybrid_engine = config.actor_rollout_ref.hybrid_engine
+
+        assert not self.hybrid_engine
+        assert self.config.data.train_batch_size == 0, "train_batch_size must be zero"
+        assert self.config.data.gen_batch_size == 1, "gen_batch_size must be one"
+        assert self.config.async_training.staleness_threshold >= 0, "staleness_threshold must larger than 0"
+        assert self.config.async_training.trigger_parameter_sync_step >= 1, (
+            "trigger_parameter_sync_step must larger than 1"
+        )
+
+        self.role_worker_mapping = role_worker_mapping
+        self.resource_pool_manager = resource_pool_manager
+        self.ray_worker_group_cls = ray_worker_group_cls
+        self.device_name = device_name if device_name else self.config.trainer.device
+        self.validation_generations_logger = ValidationGenerationsLogger(
+            project_name=self.config.trainer.project_name,
+            experiment_name=self.config.trainer.experiment_name,
+        )
+
+        self.ref_in_actor = False
+        self.kl_ctrl_in_reward = False
+        self.use_critic = False
+        self.use_reference_policy = False
+        self.use_rm = False
+
+        print("[FullyAsyncRollouter] Creating datasets...")
+        from verl.trainer.main_ppo import create_rl_dataset, create_rl_sampler
+        from verl.utils.dataset.rl_dataset import collate_fn
+
+        train_dataset = create_rl_dataset(config.data.train_files, config.data, tokenizer, processor)
+        val_dataset = create_rl_dataset(config.data.val_files, config.data, tokenizer, processor)
+        train_sampler = create_rl_sampler(config.data, train_dataset)
+
+        self._validate_config()
+        if self.config.async_training.use_trainer_do_validate:
+            rollout_gpus = config.rollout.nnodes * config.rollout.n_gpus_per_node
+            train_gpus = config.trainer.nnodes * config.trainer.n_gpus_per_node
+            total_gpus = rollout_gpus + train_gpus
+            print(f"[FullyAsyncRollouter] split before val_dataset total len: {len(val_dataset)}")
+            split_dataset = val_dataset.split(total_gpus)
+            rollout_val_dataset0 = split_dataset[:rollout_gpus]
+            from torch.utils.data import ConcatDataset
+
+            val_dataset = ConcatDataset(rollout_val_dataset0)
+            print(f"[FullyAsyncRollouter] split after val_dataset total len: {len(val_dataset)}")
+        print(f"[FullyAsyncRollouter] Rollouter _create_dataloader...\n{train_dataset}\n{val_dataset}")
+
+        self._create_dataloader(train_dataset, val_dataset, collate_fn, train_sampler)
+
+        # ==================== fully async config ====================
+
+        self.total_rollout_steps = len(self.train_dataloader) * self.config.trainer.total_epochs
+        if self.config.rollout.total_rollout_steps is not None:
+            self.total_rollout_steps = min(self.config.rollout.total_rollout_steps, self.total_rollout_steps)
+        print(f"[FullyAsyncRollouter] Total rollout steps: {self.total_rollout_steps}")
+        self.total_train_steps = None
+
+        # Rollouter parameter configuration
+        self.message_queue_client = None
+
+        # Worker groups: rollout_wg is same to actor_rollout_wg
+        self.rollout_wg = None
+        self.actor_rollout_wg = None
+        self.async_rollout_manager = None
+
+        # Config
+        self.staleness_threshold: float = config.async_training.get("staleness_threshold", 1)
+        # required_samples use ppo_mini_batch_size*require_batches as the minimum number of samples.
+        self.require_batches = config.async_training.require_batches
+        self.required_samples = config.actor_rollout_ref.actor.ppo_mini_batch_size * self.require_batches
+        self.max_required_samples = None
+        self.max_concurrent_samples = None
+        # queue size
+        self.max_queue_size = None
+
+        # Statistics
+        self.current_param_version = 0
+        self.total_generated_samples = 0
+        self.staleness_samples = 0
+        self.dropped_stale_samples = 0
+        self.processed_sample_count = 0
+        # we start from step 1
+        self.global_steps = 1
+        self.idle_start_time = None
+        self.version_start_time = None
+
+        # Concurrency control
+        # Modified by self.pause() or self._should_pause_generation()
+        self.paused = False
+        self.running = True
+        self.monitor_loop_trigger = True
+
+        # Add dataloader lock
+        self.dataloader_lock = asyncio.Lock()
+
+        # Initialize async queues
+        self.pending_queue = asyncio.Queue(maxsize=128)
+        self.active_tasks = set()
+        self.cancel_queue = asyncio.Queue()
+
+        cpu_cores = multiprocessing.cpu_count()
+        # cpu case use cpu_cores; io case use cpu_cores*2
+        self.validate_executor = ThreadPoolExecutor(max_workers=cpu_cores)
+        self.parallel_validate_and_rollout = config.async_training.get("parallel_validate_and_rollout", False)
+        self.validate_task = None
+
+    def _init_async_objects(self):
+        # Initialize asyncio synchronization primitives.
+        # We let asyncio.Condition create the Lock internally to ensure they share the same Event Loop.
+        # This avoids 'ValueError: loop argument must agree with lock' which can occur in Ray environments
+        # where the lock's captured loop (get_running_loop) differs from Condition's default loop check.
+        # Explicitly passing the loop is deprecated/removed in Python 3.10+, so this reverse-initialization
+        # is the most robust workaround.
+        self.condition = asyncio.Condition()
+        self.lock = self.condition._lock
+
+    async def set_message_queue_client(self, message_queue_client: MessageQueueClient):
+        """Set message queue client"""
+        async with self.lock:
+            self.message_queue_client = message_queue_client
+
+    async def set_max_required_samples(self):
+        async with self.lock:
+            self.max_required_samples = int(
+                self.required_samples
+                * (self.staleness_threshold + 1)
+                * self.config.async_training.trigger_parameter_sync_step
+            )
+            self.total_train_steps = int(
+                self.total_rollout_steps
+                / (self.required_samples * self.config.async_training.trigger_parameter_sync_step)
+            )
+
+            self.max_concurrent_samples = len(self.async_rollout_manager.server_handles) * 16
+            self.max_concurrent_samples = min(self.max_concurrent_samples, self.max_required_samples)
+            self.max_queue_size = self.max_required_samples
+
+            print(
+                f"[FullyAsyncRollouter] required_samples : {self.required_samples} "
+                f"max_required_samples: {self.max_required_samples} "
+                f"max_queue_size: {self.max_queue_size} "
+                f"total_train_steps: {self.total_train_steps} "
+                f"total_rollout_steps: {self.total_rollout_steps} "
+                f"max_concurrent_samples: {self.max_concurrent_samples} "
+            )
+
+    def get_rollout_wg(self):
+        """Get rollout worker group"""
+        return self.rollout_wg
+
+    def get_max_queue_size(self):
+        return self.max_queue_size
+
+    def get_total_train_steps(self):
+        return self.total_train_steps
+
+    async def update_param_version(
+        self, version: int, validate: bool = False, global_steps: int = 0, use_trainer_do_validate: bool = False
+    ):
+        """Update current parameter version"""
+        async with self.lock:
+            old_version = self.current_param_version
+            self.current_param_version = version
+            # every time param change, reset staleness_samples
+            self.staleness_samples = (
+                len(self.active_tasks) + self.cancel_queue.qsize() + await self.message_queue_client.get_queue_size()
+            )
+            timing_raw = {}
+            idle_ratio = None
+            if self.idle_start_time is not None and self.version_start_time is not None:
+                rollout_active_time = self.idle_start_time - self.version_start_time
+                rollout_version_time = time.time() - self.version_start_time
+                idle_ratio = 1 - rollout_active_time / rollout_version_time
+                timing_raw["rollouter/active_time"] = rollout_active_time
+                timing_raw["rollouter/version_time"] = rollout_version_time
+                timing_raw["rollouter/idle_ratio"] = idle_ratio
+                self.idle_start_time = None
+            print(
+                f"[FullyAsyncRollouter][Public][update_param_version] "
+                f"Parameter version updated from {old_version} to {version} "
+                f",reset staleness_samples to: {self.staleness_samples}"
+                f",idle_ratio: {idle_ratio}"
+            )
+            need_validate = (
+                (
+                    self.val_reward_fn is not None
+                    and self.config.rollout.test_freq > 0
+                    and self.current_param_version % self.config.rollout.test_freq == 0
+                    and self.current_param_version > 0
+                )  # don't test here in the initial parameter sync
+                or (validate and self.val_reward_fn is not None)
+            )
+            print(
+                f"[FullyAsyncRollouter] need_validate: {need_validate},"
+                f"parallel_validate_and_rollout: {self.parallel_validate_and_rollout}"
+            )
+            if not need_validate:
+                data = ValidateMetrics(
+                    timing_raw=timing_raw, metrics=None, global_steps=global_steps, param_version=version
+                )
+            elif need_validate and not self.parallel_validate_and_rollout:
+                data = self._validate_wrapper(timing_raw, version, global_steps, use_trainer_do_validate)
+
+            if not need_validate or not self.parallel_validate_and_rollout:
+                await self.message_queue_client.put_validate(ray.cloudpickle.dumps(data))
+
+            self.version_start_time = time.time()
+
+        if need_validate and self.parallel_validate_and_rollout:
+            if self.validate_task and not self.validate_task.done():
+                print("[FullyAsyncRollouter] validate_task is running, wait last validate_task to finish")
+                self.validate_task.get()
+            self.validate_task = asyncio.create_task(
+                self.do_validate_async(timing_raw, version, global_steps, use_trainer_do_validate)
+            )
+
+    def _validate_wrapper(
+        self, timing_raw: dict, version: int, global_steps: int = 0, use_trainer_do_validate: bool = False
+    ):
+        val_metrics = None
+        with marked_timer("rollouter/validate_time", timing_raw, color="green"):
+            val_metrics: dict = self._validate(use_trainer_do_validate)
+        data = ValidateMetrics(
+            timing_raw=timing_raw, metrics=val_metrics, global_steps=global_steps, param_version=version
+        )
+        return data
+
+    async def do_validate_async(
+        self,
+        timing_raw: dict,
+        version: int,
+        global_steps: int = 0,
+        use_trainer_do_validate: bool = False,
+    ):
+        loop = asyncio.get_running_loop()
+
+        data = await loop.run_in_executor(
+            self.validate_executor,
+            functools.partial(
+                self._validate_wrapper,
+                timing_raw=timing_raw,
+                version=version,
+                global_steps=global_steps,
+                use_trainer_do_validate=use_trainer_do_validate,
+            ),
+        )
+        await self.message_queue_client.put_validate(ray.cloudpickle.dumps(data))
+
+    async def save_checkpoint(self, local_global_step_folder: str):
+        # WARNING!: Due to the asynchronous nature, there are some in-flight samples
+        # (pending/cancel/result queue and message queue).
+        # Therefore, directly saving the state of the dataloader will result in losing these
+        # samples when resuming training.
+        # TODO: Implement dataloader recovery without losing in-flight samples.
+        from verl.utils.fs import local_mkdir_safe
+
+        # save dataloader
+        local_mkdir_safe(local_global_step_folder)
+        dataloader_local_path = os.path.join(local_global_step_folder, "data.pt")
+        async with self.dataloader_lock:
+            dataloader_state_dict = self.train_dataloader.state_dict()
+        torch.save(dataloader_state_dict, dataloader_local_path)
+        print(f"[FullyAsyncRollouter] Saved dataloader checkpoint to {dataloader_local_path}")
+
+    def load_checkpoint(self):
+        """Load checkpoint including dataloader state based on resume mode"""
+
+        if self.config.trainer.resume_mode == "disable":
+            print("[FullyAsyncRollouter] Resume mode is disabled, starting from scratch")
+            return 0
+
+        # Determine checkpoint folder path
+        if self.config.trainer.default_hdfs_dir is not None:
+            raise NotImplementedError("[FullyAsyncRollouter] Load from hdfs is not implemented yet")
+        else:
+            checkpoint_folder = self.config.trainer.default_local_dir
+            if not os.path.isabs(checkpoint_folder):
+                working_dir = os.getcwd()
+                checkpoint_folder = os.path.join(working_dir, checkpoint_folder)
+
+            global_step_folder = find_latest_ckpt_path(checkpoint_folder)
+
+        # Find and validate global_step_folder based on resume mode
+        if self.config.trainer.resume_mode == "auto":
+            if global_step_folder is None:
+                print("[FullyAsyncRollouter] Training from scratch (no checkpoint found)")
+                return 0
+        elif self.config.trainer.resume_mode == "resume_path":
+            assert isinstance(self.config.trainer.resume_from_path, str), (
+                "[FullyAsyncRollouter] resume_from_path must be str type"
+            )
+            assert "global_step_" in self.config.trainer.resume_from_path, (
+                "[FullyAsyncRollouter] resume_from_path must specify the global_steps"
+            )
+            global_step_folder = self.config.trainer.resume_from_path
+            if not os.path.isabs(global_step_folder):
+                working_dir = os.getcwd()
+                global_step_folder = os.path.join(working_dir, global_step_folder)
+        else:
+            raise ValueError(f"[FullyAsyncRollouter] Unknown resume_mode: {self.config.trainer.resume_mode}")
+
+        print(f"[FullyAsyncRollouter] Loading checkpoint from: {global_step_folder}")
+
+        # Extract and set global step
+        trainer_global_steps = int(global_step_folder.split("global_step_")[-1])
+        self.global_steps = (
+            trainer_global_steps * self.required_samples * self.config.async_training.trigger_parameter_sync_step + 1
+        )
+        print(f"[FullyAsyncRollouter] Setting global_steps to {self.global_steps}")
+
+        # Load dataloader state
+        dataloader_local_path = os.path.join(global_step_folder, "data.pt")
+        if os.path.exists(dataloader_local_path):
+            dataloader_state_dict = torch.load(dataloader_local_path, weights_only=False)
+            self.train_dataloader.load_state_dict(dataloader_state_dict)
+            print(f"[FullyAsyncRollouter] Loaded dataloader state from {dataloader_local_path}")
+        else:
+            print(
+                f"[FullyAsyncRollouter] Warning: No dataloader state found at {dataloader_local_path}, "
+                f"will start from scratch"
+            )
+
+    def _validate_config(self):
+        # Validate asynchronous training configuration
+        if not hasattr(self.config, "async_training"):
+            raise ValueError("[FullyAsyncRollouter] Missing async_training configuration")
+        assert self.config.actor_rollout_ref.rollout.calculate_log_probs, "must rollout calculate log_probs"
+
+    async def init_workers(self):
+        """Initialize distributed training workers using Ray backend.
+
+        Creates:
+        1. Ray resource pools from configuration
+        2. Worker groups for each role (actor, critic, etc.)
+        """
+        self._init_async_objects()
+        self._init_resource_pools()
+        self._create_worker_classes()
+        self._init_worker_groups()
+        self._init_models()
+        await self._init_async_rollout_manager()
+
+    def _create_actor_rollout_classes(self):
+        # only create rollout
+        for role in [Role.Rollout]:
+            resource_pool = self.resource_pool_manager.get_resource_pool(role)
+            role_cls = RayClassWithInitArgs(
+                cls=self.role_worker_mapping[role],
+                config=self.config.actor_rollout_ref,
+                role=str(role),
+            )
+            self.resource_pool_to_cls[resource_pool][str(role)] = role_cls
+
+    def _init_models(self):
+        self.rollout_wg = self.all_wg[str(Role.Rollout)]
+        self.rollout_wg.init_model()
+        self.actor_rollout_wg = self.rollout_wg
+
+    def _create_continuous_iterator(self):
+        """
+        Create a continuous data iterator across epoch
+        """
+        for epoch in range(self.config.rollout.total_epochs):
+            iterator = iter(self.train_dataloader)
+            for batch_dict in iterator:
+                yield epoch, batch_dict
+
+    async def _init_async_rollout_manager(self):
+        # create async rollout manager and request scheduler
+        assert self.config.actor_rollout_ref.rollout.mode == "async"
+        from verl.experimental.fully_async_policy.agent_loop import FullyAsyncAgentLoopManager
+
+        self.async_rollout_mode = True
+        self.async_rollout_manager = await FullyAsyncAgentLoopManager.create(
+            config=self.config,
+            worker_group=self.rollout_wg,
+        )
+
+    # Add samples to the pending_queue
+    async def _feed_samples(self):
+        continuous_iterator = self._create_continuous_iterator()
+
+        for epoch, batch_dict in continuous_iterator:
+            # Similar to _prepare_generate_batch: Separate data
+            full_batch = prepare_single_generation_data(batch_dict, self.config)
+
+            sample_id = f"sample_{epoch}_{self.global_steps}"
+
+            rollout_sample = RolloutSample(
+                full_batch=full_batch,
+                agent_loop_output_list=[None] * self.config.actor_rollout_ref.rollout.n,
+                sample_id=sample_id,
+                epoch=epoch,
+                param_version=0,
+                param_version_start=[],
+                param_version_end=[],
+                processing_times=[],
+                tool_calls=[],
+                rollout_status={},
+            )
+
+            await self.pending_queue.put(rollout_sample)
+
+            # Check if have reached the last step
+            if self.global_steps >= self.total_rollout_steps:
+                print(
+                    f"[FullyAsyncRollouter][Feed] "
+                    f"Maximum count has been reached, stop adding new samples"
+                    f"{self.global_steps} >= {self.total_rollout_steps}"
+                )
+                break
+
+            self.global_steps += 1
+
+        # End signal
+        await self.pending_queue.put("DONE")
+        print(f"[FullyAsyncRollouter][Feed] Sample addition is complete, {self.global_steps} samples have been added")
+
+    async def _processor_worker(self):
+        """
+        Streaming worker coroutines, a sample is submitted for processing without waiting for batches
+        """
+        while True:
+            if self.paused or await self._should_pause_generation():
+                print(
+                    "[FullyAsyncRollouter][Processor] Received pause signal, waiting for remaining tasks to return..."
+                )
+                async with self.lock:
+                    self.paused = True
+                while self.active_tasks:
+                    async with self.lock:
+                        # After acquiring the lock, the number of active_tasks may change, need to be verified again
+                        if self.active_tasks:
+                            done_tasks, self.active_tasks = await asyncio.wait(
+                                self.active_tasks, return_when=asyncio.FIRST_COMPLETED
+                            )
+                        for task in done_tasks:
+                            await task
+
+                async with self.lock:
+                    while self.paused:
+                        self.idle_start_time = time.time()
+                        await self.condition.wait()
+                continue
+
+            simple_from_cancel_queue = False
+            if not self.cancel_queue.empty():
+                rollout_sample = await self.cancel_queue.get()
+                simple_from_cancel_queue = True
+            else:
+                rollout_sample = await self.pending_queue.get()
+                self.staleness_samples += 1
+
+            if rollout_sample == "DONE":
+                print(
+                    "[FullyAsyncRollouter][Processor] Received end signal, waiting for remaining tasks to complete..."
+                )
+                while self.active_tasks:
+                    async with self.lock:
+                        if self.active_tasks:
+                            done_tasks, self.active_tasks = await asyncio.wait(
+                                self.active_tasks, return_when=asyncio.FIRST_COMPLETED
+                            )
+                        for task in done_tasks:
+                            await task
+                break
+
+            # Check whether the number of concurrent tasks exceeds the limit
+            while len(self.active_tasks) >= self.max_concurrent_samples:
+                async with self.lock:
+                    if self.active_tasks:
+                        done_tasks, self.active_tasks = await asyncio.wait(
+                            self.active_tasks, return_when=asyncio.FIRST_COMPLETED
+                        )
+                    for task in done_tasks:
+                        await task
+
+            # Submit single sample processing
+            async with self.lock:
+                # After the pause is over, the lock is acquired and it is necessary
+                # to determine whether it is the pause phase, otherwise continue to wait
+                while self.paused:
+                    await self.condition.wait()
+                task = asyncio.create_task(
+                    self._process_single_sample_streaming(rollout_sample),
+                    name=rollout_sample.sample_id,
+                )
+                self.active_tasks.add(task)
+
+            if simple_from_cancel_queue:
+                self.cancel_queue.task_done()
+            else:
+                self.pending_queue.task_done()
+
+    async def _process_single_sample_streaming(self, rollout_sample: RolloutSample):
+        """Process a single sample streamingly"""
+        # Calling asynchronous generation methods
+        rollout_sample.full_batch.non_tensor_batch["param_version"] = [self.current_param_version] * len(
+            rollout_sample.full_batch
+        )
+        ret, is_cancel = await self.async_rollout_manager.generate_single_sample_async(
+            rollout_sample.full_batch, rollout_sample.agent_loop_output_list
+        )
+        if not is_cancel:
+            rollout_sample.full_batch = ret
+            rollout_sample.full_batch.non_tensor_batch["uid"] = np.array(
+                [f"uid_{rollout_sample.sample_id}"] * len(rollout_sample.full_batch), dtype=object
+            )
+            rollout_sample.param_version = self.current_param_version
+            rollout_sample.rollout_status = await self.get_statistics()
+            rollout_sample.agent_loop_output_list = []
+
+            success = await self.message_queue_client.put_sample(
+                sample=ray.cloudpickle.dumps(rollout_sample),
+                param_version=rollout_sample.param_version,
+            )
+            if success:
+                self.total_generated_samples += 1
+            else:
+                self.dropped_stale_samples += 1
+        else:
+            rollout_sample.agent_loop_output_list = ret
+            await self.cancel_queue.put(rollout_sample)
+
+        self.processed_sample_count += 1
+
+    async def _streaming_generation_main(self):
+        """The main entry method for stream processing"""
+
+        if self.async_rollout_manager is None:
+            await self._init_async_rollout_manager()
+
+        # Start the streaming loop
+        print(f"[FullyAsyncRollouter] Start streaming mode, maximum concurrent samples: {self.max_concurrent_samples}")
+
+        # Start sample feed coroutine, streaming process coroutine
+        self.feed_task = asyncio.create_task(self._feed_samples())
+        self.processor_task = asyncio.create_task(self._processor_worker())
+
+        try:
+            # Wait for sample feed to complete
+            # Use asyncio.wait to monitor all tasks. If processor exits early,
+            # detect it instead of blocking on feed_task (it might be stuck on a full queue).
+            done, pending = await asyncio.wait(
+                [self.feed_task, self.processor_task], return_when=asyncio.FIRST_COMPLETED
+            )
+
+            for task in done:
+                if task.exception():
+                    raise task.exception()
+
+            if self.feed_task not in done:
+                raise RuntimeError("Processor task exited prematurely")
+
+            print("[FullyAsyncRollouter] Sample feed completed")
+
+            # Wait for streaming to complete
+            await self.processor_task
+            print("[FullyAsyncRollouter] Streaming process completed")
+
+        except Exception as e:
+            print(f"[FullyAsyncRollouter] Streaming process exception:{e}")
+
+        finally:
+            if self.processor_task:
+                self.processor_task.cancel()
+
+            await asyncio.gather(self.processor_task, return_exceptions=True)
+
+        # Send a finish signal
+        await self.message_queue_client.put_sample(
+            sample=None,
+            param_version=self.current_param_version,
+        )
+
+        async with self.lock:
+            self.running = False
+
+    async def fit(self):
+        """
+        Start the async rollouter - entry point that sets up and runs async tasks
+        Main async fit method that coordinates all coroutines
+        """
+
+        print("[FullyAsyncRollouter] Starting FullyAsyncRollouter...")
+
+        if self.message_queue_client is None:
+            raise ValueError("MessageQueue client not set. Call set_message_queue_client() first.")
+
+        # Set the running status flag
+        async with self.lock:
+            self.paused = False
+            self.running = True
+
+        # Create the main asynchronous task
+        generation_task = asyncio.create_task(self._streaming_generation_main())
+        monitor_task = asyncio.create_task(self._async_monitor_loop())
+
+        try:
+            # Run build and monitoring tasks concurrently
+            await asyncio.gather(generation_task, monitor_task, return_exceptions=True)
+        except Exception as e:
+            print(f"[FullyAsyncRollouter] Asynchronous task execution error: {e}")
+        finally:
+            if not generation_task.done():
+                generation_task.cancel()
+            if not monitor_task.done():
+                monitor_task.cancel()
+
+            # Wait for the task to complete
+            await asyncio.gather(generation_task, monitor_task, return_exceptions=True)
+
+        print("[FullyAsyncRollouter] Rollouter fit completed")
+
+    async def _async_monitor_loop(self):
+        """
+        Async coroutine for monitoring:
+        Function 1: Log information output
+        Function 2: Trigger rollout recovery
+        """
+        last_stats_time = time.time()
+        stats_interval = 60.0
+        check_interval = 10.0
+
+        while True:
+            async with self.lock:
+                if not self.running:
+                    break
+            await asyncio.sleep(check_interval)
+            # Print statistics periodically
+            current_time = time.time()
+            if current_time - last_stats_time >= stats_interval:
+                stats = await self.get_statistics()
+                print(f"[FullyAsyncRollouter][MonitorLoop][Statistics] {pformat(stats)}")
+                last_stats_time = current_time
+
+            # Trigger rollout recovery
+            if self.monitor_loop_trigger:
+                if not await self._should_pause_generation():
+                    async with self.lock:
+                        self.paused = False
+                        self.condition.notify_all()
+
+    async def _should_pause_generation(self) -> bool:
+        """Determine whether the build should be paused"""
+        queue_stats = self.message_queue_client.get_statistics_sync()
+        queue_size = queue_stats["queue_size"]
+
+        if queue_size >= self.max_queue_size:
+            if not self.paused:
+                print(
+                    f"[FullyAsyncRollouter][ShouldPause]  "
+                    f"due to full queue: size={queue_size}, max={self.max_queue_size}"
+                )
+            return True
+
+        if self.staleness_samples >= self.max_required_samples:
+            if not self.paused:
+                print(
+                    "[FullyAsyncRollouter][ShouldPause] "
+                    f"due to "
+                    f"staleness_samples {self.staleness_samples} >= max_required_samples {self.max_required_samples} "
+                )
+            return True
+
+        return False
+
+    async def pause(self):
+        """pause rollout"""
+        print("[FullyAsyncRollouter][Public][Pause] partial rollout:", self.config.async_training.partial_rollout)
+        async with self.lock:
+            self.paused = True
+            # Cancel all rollout tasks
+            if self.config.async_training.partial_rollout:
+                await self.async_rollout_manager.cancel()
+                print("[FullyAsyncRollouter][Public][Pause] Unfinished rollout tasks canceled")
+            if self.active_tasks:
+                await asyncio.gather(*self.active_tasks, return_exceptions=True)
+                self.active_tasks.clear()
+                print("[FullyAsyncRollouter][Public][Pause] All active tasks completed")
+            print("[FullyAsyncRollouter][Public][Pause] Prefix cache reset")
+            # Always clear KV cache to release GPU memory during weight synchronization,
+            # regardless of partial_rollout setting.
+            await self.async_rollout_manager.clear_kv_cache()
+            self.monitor_loop_trigger = False
+
+    async def resume(self, dependency_ref: ObjectRef = None):
+        if dependency_ref is not None:
+            ray.get(dependency_ref)
+        print("[FullyAsyncRollouter][Public][Resume]")
+        async with self.lock:
+            if self.config.async_training.partial_rollout:
+                await self.async_rollout_manager.resume()
+            self.paused = False
+            self.monitor_loop_trigger = True
+            self.condition.notify_all()
+
+    async def get_statistics(self) -> dict:
+        queue_stats = self.message_queue_client.get_statistics_sync()
+
+        stats = {
+            # monitor stats
+            "monitor/active_tasks_size": len(self.active_tasks),
+            "monitor/queue/pending_queue_size": self.pending_queue.qsize(),
+            "monitor/queue/cancel_queue_size": self.cancel_queue.qsize(),
+            "monitor/queue/mq_queue_size": queue_stats["queue_size"],
+            # counting stats
+            "count/current_param_version": self.current_param_version,
+            "count/total_generated_samples": self.total_generated_samples,
+            "count/staleness_samples": self.staleness_samples,
+            "count/dropped_stale_samples": self.dropped_stale_samples,
+            # static stats
+            "static/max_required_samples": self.max_required_samples,
+            "static/required_samples": self.required_samples,
+            "static/staleness_threshold": self.staleness_threshold,
+            "static/max_queue_size": self.max_queue_size,
+            "static/max_concurrent_samples": self.max_concurrent_samples,
+        }
+
+        return stats

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/fully_async_trainer.py

@@ -0,0 +1,612 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+from datetime import datetime
+from pprint import pprint
+from typing import Any
+
+import ray
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from verl.experimental.fully_async_policy.detach_utils import (
+    MetricsAggregator,
+    ValidateMetrics,
+    assemble_batch_from_rollout_samples,
+)
+from verl.experimental.fully_async_policy.message_queue import MessageQueueClient
+from verl.experimental.fully_async_policy.ray_trainer import FullyAsyncRayPPOTrainer
+from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup
+from verl.trainer.ppo import core_algos
+from verl.trainer.ppo.ray_trainer import ResourcePoolManager
+from verl.trainer.ppo.reward import load_reward_manager
+from verl.trainer.ppo.utils import Role, WorkerType, need_critic, need_reference_policy, need_reward_model
+from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path, should_save_ckpt_esi
+from verl.utils.debug import marked_timer
+
+
+@ray.remote(num_cpus=10)
+class FullyAsyncTrainer(FullyAsyncRayPPOTrainer):
+    """
+    A fully asynchronous PPO trainer that obtains samples from a MessageQueue for training.
+    Based on an improved implementation of OneStepOffRayTrainer
+    """
+
+    def __init__(
+        self,
+        config,
+        tokenizer,
+        role_worker_mapping: dict[Role, WorkerType],
+        resource_pool_manager: ResourcePoolManager,
+        ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup,
+        processor=None,
+        reward_fn=None,
+        val_reward_fn=None,
+        device_name=None,
+    ):
+        # Store the tokenizer for text processing
+        self.tokenizer = tokenizer
+        self.processor = processor
+        self.config = config
+        self.reward_fn = load_reward_manager(
+            config, tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {})
+        )
+        self.val_reward_fn = load_reward_manager(
+            config, tokenizer, num_examine=1, **config.reward_model.get("reward_kwargs", {})
+        )
+
+        self.hybrid_engine = config.actor_rollout_ref.hybrid_engine
+        assert not self.hybrid_engine
+
+        self.role_worker_mapping = role_worker_mapping
+        self.resource_pool_manager = resource_pool_manager
+        self.use_reference_policy = need_reference_policy(self.config)
+        self.use_rm = need_reward_model(self.role_worker_mapping)
+        self.use_critic = need_critic(self.config)
+        self.ray_worker_group_cls = ray_worker_group_cls
+        self.device_name = device_name if device_name else self.config.trainer.device
+
+        lora_rank = config.actor_rollout_ref.model.get("lora", {}).get("rank", 0)
+        if lora_rank <= 0:
+            lora_rank = config.actor_rollout_ref.model.get("lora_rank", 0)
+        # if ref_in_actor is True, the reference policy will be actor without lora applied
+        self.ref_in_actor = lora_rank > 0
+
+        # define in-reward KL control
+        # kl loss control currently not suppoorted
+        if self.config.algorithm.use_kl_in_reward:
+            self.kl_ctrl_in_reward = core_algos.get_kl_controller(self.config.algorithm.kl_ctrl)
+
+        # ==================== fully async config ====================
+
+        self.message_queue_client = None
+        self.param_synchronizer = None
+
+        # Statistics
+        # we start from step 1
+        self.global_steps = 1
+        self.local_trigger_step = 1
+        self.processed_samples = 0
+        self.stale_samples_processed = 0
+        self.stale_trajectory_processed = 0
+        self.current_param_version = 0
+        self.total_train_steps = None
+        self.progress_bar = None
+        self.trigger_parameter_sync_step = config.async_training.trigger_parameter_sync_step
+        self.last_ckpt_version = 0
+        self.train_val_metrics = None
+        self.train_role = Role.ActorRollout if config.async_training.use_trainer_do_validate else Role.Actor
+
+        # required_samples use ppo_mini_batch_size*require_batches as the minimum number of samples.
+        self.require_batches = config.async_training.require_batches
+        self.required_samples = config.actor_rollout_ref.actor.ppo_mini_batch_size * self.require_batches
+        self.compute_prox_log_prob = self.config.async_training.compute_prox_log_prob
+        total_gpus = (
+            config.trainer.nnodes * config.trainer.n_gpus_per_node
+            + config.rollout.nnodes * config.rollout.n_gpus_per_node
+        )
+        self.metrics_aggregator = MetricsAggregator(total_gpus=total_gpus)
+
+        # use trainer to do validation
+        if self.config.async_training.use_trainer_do_validate:
+            from verl.trainer.main_ppo import create_rl_dataset
+            from verl.utils.dataset.rl_dataset import collate_fn
+
+            val_dataset = create_rl_dataset(config.data.val_files, config.data, tokenizer, processor)
+            rollout_gpus = config.rollout.nnodes * config.rollout.n_gpus_per_node
+            print(f"[FullyAsyncTrainer] split before val_dataset total len: {len(val_dataset)}")
+            split_dataset = val_dataset.split(total_gpus)
+            rollout_val_dataset0 = split_dataset[rollout_gpus:]
+            from torch.utils.data import ConcatDataset
+
+            val_dataset = ConcatDataset(rollout_val_dataset0)
+            print(f"[FullyAsyncTrainer] split after val_dataset total len: {len(val_dataset)}")
+            self.val_dataset = val_dataset
+            # update val_dataloader
+            val_batch_size = self.config.data.val_batch_size  # Prefer config value if set
+            if val_batch_size is None:
+                val_batch_size = len(val_dataset)
+            from torchdata.stateful_dataloader import StatefulDataLoader
+
+            print(f"[FullyAsyncTrainer] create val_dataloader with batch_size: {val_batch_size}")
+            self.val_dataloader = StatefulDataLoader(
+                dataset=val_dataset,
+                batch_size=val_batch_size,
+                num_workers=self.config.data["dataloader_num_workers"],
+                shuffle=self.config.data.get("validation_shuffle", True),
+                drop_last=False,
+                collate_fn=collate_fn,
+            )
+
+    def set_message_queue_client(self, message_queue_client: MessageQueueClient):
+        """Set message queue client"""
+        self.message_queue_client = message_queue_client
+
+    def set_parameter_synchronizer(self, param_synchronizer):
+        """Set parameter synchronizer"""
+        self.param_synchronizer = param_synchronizer
+
+    def set_total_train_steps(self, total_train_steps):
+        self.total_train_steps = total_train_steps
+        self.progress_bar = tqdm(total=self.total_train_steps, initial=0, desc="Training Progress")
+
+    def get_actor_wg(self):
+        """Get actor worker group"""
+        return self.actor_wg
+
+    def _get_samples_from_queue(self) -> tuple[None, None] | tuple[int, Any]:
+        """
+        Get samples from message queue and compose gen_batch_output
+        Uses a loop to continuously collect samples until enough are gathered
+
+        Returns:
+            tuple: (epoch, batch_dict, gen_batch_output)
+        """
+        print(
+            f"[FullyAsyncTrainer] Requesting {self.required_samples} samples from queue",
+            flush=True,
+        )
+
+        # Collect samples using a simple loop calling get_sample
+        consumer_start = time.time()
+        queue_samples = []
+        queue_len = 0
+        while len(queue_samples) < self.required_samples:
+            # Get a single sample and wait until there is a sample or None is received
+            sample, queue_len = self.message_queue_client.get_sample_sync()
+
+            if sample is None:
+                print(
+                    f"[FullyAsyncTrainer] Detected termination signal (None), stopping sample collection. "
+                    f"Collected {len(queue_samples)}/{self.required_samples} samples"
+                )
+                break
+
+            queue_samples.append(sample)
+
+            if len(queue_samples) % 64 == 0:
+                print(
+                    f"[FullyAsyncTrainer] Collected {len(queue_samples)}/{self.required_samples} samples. "
+                    f"mq_len: {queue_len}"
+                )
+
+        consumer_end = time.time()
+
+        if not queue_samples or len(queue_samples) < self.required_samples:
+            print("[FullyAsyncTrainer] not enough samples collected after loop")
+            return None, None
+        total_wait_time = consumer_end - consumer_start
+
+        print(
+            f"[FullyAsyncTrainer] Loop collection completed: {len(queue_samples)}/{self.required_samples} samples, "
+            f"total wait time: {total_wait_time:.2f} seconds."
+            f"mq_len: {queue_len}"
+        )
+
+        queue_samples = [ray.cloudpickle.loads(x) for x in queue_samples]
+        # Assemble batch - now working directly with RolloutSample objects
+        if self.config.trainer.balance_batch:
+            batch = assemble_batch_from_rollout_samples(queue_samples, self.tokenizer, self.config, self._balance_batch)
+        else:
+            batch = assemble_batch_from_rollout_samples(queue_samples, self.tokenizer, self.config, None)
+
+        batch.meta_info["fully_async/total_wait_time"] = total_wait_time
+        return 0, batch
+
+    def _create_actor_rollout_classes(self):
+        # create actor
+        for role in [self.train_role]:
+            resource_pool = self.resource_pool_manager.get_resource_pool(role)
+            role_cls = RayClassWithInitArgs(
+                cls=self.role_worker_mapping[role],
+                config=self.config.actor_rollout_ref,
+                role=str(role),
+            )
+            self.resource_pool_to_cls[resource_pool][str(role)] = role_cls
+
+    def _init_models(self):
+        if self.use_critic:
+            self.critic_wg = self.all_wg[str(Role.Critic)]
+            self.critic_wg.init_model()
+
+        if self.use_reference_policy and not self.ref_in_actor:
+            self.ref_policy_wg = self.all_wg[str(Role.RefPolicy)]
+            self.ref_policy_wg.init_model()
+
+        if self.use_rm:
+            self.rm_wg = self.all_wg[str(Role.RewardModel)]
+            self.rm_wg.init_model()
+
+        self.actor_wg = self.all_wg[str(self.train_role)]
+        self.actor_wg.init_model()
+        self.actor_rollout_wg = self.actor_wg  # to be compatible with the functions that not be modified
+
+    async def init_workers(self):
+        """Initialize distributed training workers using Ray backend.
+        Creates:
+        1. Ray resource pools from configuration
+        2. Worker groups for each role (actor, critic, etc.)
+        """
+        # self._init_async_objects()
+        self._init_resource_pools()
+        self._create_worker_classes()
+        self._init_worker_groups()
+        self._init_models()
+        await self._init_async_rollout_manager()
+
+    async def _init_async_rollout_manager(self):
+        # use async rollout do validate
+        print(f"[FullyAsyncTrainer] use_trainer_do_validate: {self.config.async_training.use_trainer_do_validate}")
+        if self.config.async_training.use_trainer_do_validate:
+            assert self.config.actor_rollout_ref.rollout.mode == "async"
+            self.async_rollout_mode = True
+            print("[FullyAsyncTrainer] Init async rollout manager")
+            from verl.experimental.fully_async_policy.agent_loop import FullyAsyncAgentLoopManager
+
+            self.async_rollout_manager = await FullyAsyncAgentLoopManager.create(
+                config=self.config, worker_group=self.actor_rollout_wg
+            )
+            print("[FullyAsyncTrainer] async_rollout_manager sleep")
+            await self.async_rollout_manager.sleep()
+        else:
+            print("[FullyAsyncTrainer] Skip async rollout manager (use_trainer_do_validate=False)")
+
+    async def fit(self):
+        """
+        The training loop of PPO.
+        The driver process only need to call the compute functions of the worker group through RPC
+        to construct the PPO dataflow.
+        The light-weight advantage computation is done on the driver process.
+        """
+        print("[FullyAsyncTrainer] Starting FullyAsyncTrainer...")
+        if self.message_queue_client is None:
+            raise ValueError("MessageQueue client not set. Call set_message_queue_client() first.")
+        if self.param_synchronizer is None:
+            raise ValueError("param_synchronizer client not set. Call set_parameter_synchronizer() first.")
+
+        from verl.utils.tracking import Tracking
+
+        self.logger = Tracking(
+            project_name=self.config.trainer.project_name,
+            experiment_name=self.config.trainer.experiment_name,
+            default_backend=self.config.trainer.logger,
+            config=OmegaConf.to_container(self.config, resolve=True),
+        )
+
+        self.max_steps_duration = 0
+
+        # get validate data before training
+        self._log_validation_data()
+
+        # Use queue mode, no need for traditional dataloader iterator
+        # Initialize to get the first batch of data
+        while True:
+            metrics = {}
+            timing_raw = {}
+
+            with marked_timer("step", timing_raw):
+                with marked_timer("gen", timing_raw, color="red"):
+                    epoch, batch = self._get_samples_from_queue()
+                    if batch is None:
+                        break
+                    self._collect_metrics_from_samples(batch, metrics)
+                batch, reward_extra_infos_dict = self._process_batch_common(
+                    batch, metrics, timing_raw, self.local_trigger_step if self.compute_prox_log_prob else None
+                )
+                self._log_rollout(batch, reward_extra_infos_dict, timing_raw)
+
+            self._collect_metrics(batch, 0, metrics, timing_raw)
+            self.metrics_aggregator.add_step_metrics(
+                metrics=metrics, sample_count=self.required_samples, timestamp=time.time()
+            )
+            # Trigger parameter synchronization after training step
+            time_str = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+            print(
+                f"[FullyAsyncTrainer] global_steps: {self.global_steps} "
+                f"local_trigger_step: {self.local_trigger_step} "
+                f"trigger_parameter_sync_step: {self.trigger_parameter_sync_step} "
+                f"{time_str}"
+            )
+            await self._trigger_parameter_sync_after_step(global_steps=self.global_steps)
+            self._log_validation_data()
+            self._check_save_checkpoint(timing_raw)
+            self.global_steps += 1
+
+        # final parameter sync and validate
+        # 1. waiting remaining validate task
+        ray.get(self.param_synchronizer.wait_last_valid.remote())
+        self._log_validation_data()
+        # 2. perform addtional parameter_sync and validate if trainer already updated
+        if self.current_param_version % self.config.rollout.test_freq != 0 or self.local_trigger_step > 1:
+            await self._trigger_parameter_sync_after_step(validate=True, global_steps=self.global_steps)
+            ray.get(self.param_synchronizer.wait_last_valid.remote())
+            self._log_validation_data()
+        self.progress_bar.close()
+
+        self._check_save_checkpoint(timing_raw)
+
+    def _check_save_checkpoint(self, timing_raw):
+        if self.current_param_version == self.last_ckpt_version:
+            return
+        # Check if the ESI (Elastic Server Instance)/training plan is close to expiration.
+        esi_close_to_expiration = should_save_ckpt_esi(
+            max_steps_duration=self.max_steps_duration,
+            redundant_time=self.config.trainer.esi_redundant_time,
+        )
+        # Check if the conditions for saving a checkpoint are met.
+        # The conditions include a mandatory condition (1) and
+        # one of the following optional conditions (2/3/4):
+        # 1. The save frequency is set to a positive value.
+        # 2. The current step number is a multiple of the save frequency.
+        # 3. The ESI(Elastic Server Instance)/training plan is close to expiration.
+        if self.config.trainer.save_freq > 0 and (
+            self.current_param_version % self.config.trainer.save_freq == 0 or esi_close_to_expiration
+        ):
+            if esi_close_to_expiration:
+                print("Force saving checkpoint: ESI instance expiration approaching.")
+            with marked_timer("save_checkpoint", timing_raw, color="green"):
+                self._save_checkpoint()
+                self.last_ckpt_version = self.current_param_version
+
+    def _save_checkpoint(self):
+        # Warning: Currently, to align the training process and metrics of colocate,
+        # we use current_param_version instead of global step.
+        # This can be logically aligned with the original self.global_steps of colocate
+        # and is used for metrics and ckpt. which means that the parameter synchronization
+        # from trainer to rollouter will increase by 1 each time.
+
+        # path: given_path + `/global_step_{global_steps}` + `/actor`
+        local_global_step_folder = os.path.join(
+            self.config.trainer.default_local_dir, f"global_step_{self.current_param_version}"
+        )
+
+        print(f"[FullyAsyncTrainer] local_global_step_folder: {local_global_step_folder}")
+        actor_local_path = os.path.join(local_global_step_folder, "actor")
+
+        actor_remote_path = (
+            None
+            if self.config.trainer.default_hdfs_dir is None
+            else os.path.join(
+                self.config.trainer.default_hdfs_dir, f"global_step_{self.current_param_version}", "actor"
+            )
+        )
+
+        remove_previous_ckpt_in_save = self.config.trainer.get("remove_previous_ckpt_in_save", False)
+        if remove_previous_ckpt_in_save:
+            print(
+                "[FullyAsyncTrainer] Warning: remove_previous_ckpt_in_save is deprecated,"
+                + " set max_actor_ckpt_to_keep=1 and max_critic_ckpt_to_keep=1 instead"
+            )
+        max_actor_ckpt_to_keep = (
+            self.config.trainer.get("max_actor_ckpt_to_keep", None) if not remove_previous_ckpt_in_save else 1
+        )
+        max_critic_ckpt_to_keep = (
+            self.config.trainer.get("max_critic_ckpt_to_keep", None) if not remove_previous_ckpt_in_save else 1
+        )
+
+        self.actor_rollout_wg.save_checkpoint(
+            actor_local_path, actor_remote_path, self.current_param_version, max_ckpt_to_keep=max_actor_ckpt_to_keep
+        )
+
+        if self.use_critic:
+            critic_local_path = os.path.join(local_global_step_folder, str(Role.Critic))
+            critic_remote_path = (
+                None
+                if self.config.trainer.default_hdfs_dir is None
+                else os.path.join(
+                    self.config.trainer.default_hdfs_dir, f"global_step_{self.current_param_version}", str(Role.Critic)
+                )
+            )
+            self.critic_wg.save_checkpoint(
+                critic_local_path,
+                critic_remote_path,
+                self.current_param_version,
+                max_ckpt_to_keep=max_critic_ckpt_to_keep,
+            )
+        ray.get(self.param_synchronizer.rollouter_save_checkpoint.remote(local_global_step_folder))
+        # latest checkpointed iteration tracker (for atomic usage)
+        local_latest_checkpointed_iteration = os.path.join(
+            self.config.trainer.default_local_dir, "latest_checkpointed_iteration.txt"
+        )
+        with open(local_latest_checkpointed_iteration, "w") as f:
+            f.write(str(self.current_param_version))
+
+    def load_checkpoint(self):
+        if self.config.trainer.resume_mode == "disable":
+            # NOTE: while there is no checkpoint to load, we still need to offload the model and optimizer to CPU
+            self.actor_rollout_wg.load_checkpoint(None)
+            return 0
+
+        # load from hdfs
+        if self.config.trainer.default_hdfs_dir is not None:
+            raise NotImplementedError("load from hdfs is not implemented yet")
+        else:
+            checkpoint_folder = self.config.trainer.default_local_dir  # TODO: check path
+            if not os.path.isabs(checkpoint_folder):
+                working_dir = os.getcwd()
+                checkpoint_folder = os.path.join(working_dir, checkpoint_folder)
+            global_step_folder = find_latest_ckpt_path(checkpoint_folder)  # None if no latest
+
+        # find global_step_folder
+        if self.config.trainer.resume_mode == "auto":
+            if global_step_folder is None:
+                print("[FullyAsyncTrainer] Training from scratch")
+                self.actor_rollout_wg.load_checkpoint(None)
+                return 0
+        else:
+            if self.config.trainer.resume_mode == "resume_path":
+                assert isinstance(self.config.trainer.resume_from_path, str), "resume ckpt must be str type"
+                assert "global_step_" in self.config.trainer.resume_from_path, (
+                    "resume ckpt must specify the global_steps"
+                )
+                global_step_folder = self.config.trainer.resume_from_path
+                if not os.path.isabs(global_step_folder):
+                    working_dir = os.getcwd()
+                    global_step_folder = os.path.join(working_dir, global_step_folder)
+        print(f"[FullyAsyncTrainer] Load from checkpoint folder: {global_step_folder}")
+        # set global step
+        self.current_param_version = int(global_step_folder.split("global_step_")[-1])
+        self.global_steps = self.current_param_version * self.trigger_parameter_sync_step + 1
+        self.last_ckpt_version = self.current_param_version
+        print(
+            f"[FullyAsyncTrainer] Setting global step to {self.global_steps}, "
+            f"current_param_version to {self.current_param_version}"
+        )
+        print(f"[FullyAsyncTrainer] Resuming from  {global_step_folder}")
+
+        actor_path = os.path.join(global_step_folder, "actor")
+        critic_path = os.path.join(global_step_folder, str(Role.Critic))
+        # load actor
+        self.actor_rollout_wg.load_checkpoint(
+            actor_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load
+        )
+        # load critic
+        if self.use_critic:
+            self.critic_wg.load_checkpoint(
+                critic_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load
+            )
+        return self.current_param_version
+
+    def _collect_metrics_from_samples(self, batch, metrics):
+        """
+        Collect metrics from samples
+        """
+        if hasattr(batch, "meta_info") and batch.meta_info:
+            samples_param_versions = batch.meta_info["rollout_param_versions"]
+            stale_count = sum(1 for v in samples_param_versions if self.current_param_version - v >= 1)
+            self.stale_samples_processed += stale_count
+            trajectory_param_versions = batch.meta_info["trajectory_param_versions"]
+            stale_traj_count = sum(1 for v in trajectory_param_versions if self.current_param_version - v >= 1)
+            self.stale_trajectory_processed += stale_traj_count
+            metrics.update(
+                {
+                    "fully_async/count/stale_samples_processed": self.stale_samples_processed,
+                    "fully_async/count/stale_trajectory_processed": self.stale_trajectory_processed,
+                    "fully_async/count/current_param_version": self.current_param_version,
+                }
+            )
+            for key, value in batch.meta_info.items():
+                if key.startswith("fully_async") or key.startswith("timing_s"):
+                    metrics[key] = value
+
+    async def _trigger_parameter_sync_after_step(self, validate: bool = False, global_steps: int = None):
+        """
+        Trigger parameter synchronization after training step
+        This ensures rollouter always uses the latest trained parameters
+        """
+        if self.local_trigger_step < self.trigger_parameter_sync_step and not validate:
+            self.local_trigger_step += 1
+            return
+
+        self.current_param_version += 1
+        self.local_trigger_step = 1
+        self.logger.log(
+            data=self.metrics_aggregator.get_aggregated_metrics(),
+            step=self.current_param_version,
+        )
+        self.progress_bar.update(1)
+        self.metrics_aggregator.reset()
+        timing_param_sync = {}
+        with marked_timer("timing_s/wait_last_valid", timing_param_sync):
+            ray.get(self.param_synchronizer.wait_last_valid.remote())
+        with marked_timer("timing_s/param_sync", timing_param_sync):
+            ray.get(
+                self.param_synchronizer.sync_weights.remote(
+                    self.current_param_version,
+                    validate=validate,
+                    global_steps=global_steps,
+                    use_trainer_do_validate=self.config.async_training.use_trainer_do_validate,
+                )
+            )
+
+        #  do trainer validate
+        do_validate_param = (
+            self.config.rollout.test_freq > 0
+            and self.current_param_version % self.config.rollout.test_freq == 0
+            and self.current_param_version > 0
+        )
+        print(f"do_validate_param: {do_validate_param}")
+        if do_validate_param and self.reward_fn is not None and self.config.async_training.use_trainer_do_validate:
+            print(f"[FullyAsyncTrainer] validate param version: {self.current_param_version}")
+            await self._validate_process()
+        else:
+            self.train_val_metrics = None
+        self.logger.log(data=timing_param_sync, step=self.current_param_version)
+
+    def _log_validation_data(self):
+        """
+        Log validation data
+        """
+        val_data = self.message_queue_client.get_validate_sync()
+        if not val_data:
+            return
+
+        val_metrics: ValidateMetrics = ray.cloudpickle.loads(val_data)
+        if self.train_val_metrics and self.config.async_training.use_trainer_do_validate:
+            # merge info
+            timing_param_sync = {}
+            with marked_timer("timing_s/merge_val", timing_param_sync):
+                new_metrics = self._merge_validation_results(self.train_val_metrics, val_metrics.metrics)
+            if new_metrics:
+                self.logger.log(data=new_metrics, step=val_metrics.param_version)
+                pprint(
+                    f"[FullyAsyncTrainer] parameter version: {val_metrics.param_version} "
+                    f"Validation metrics: {new_metrics}, timing_param_sync: {timing_param_sync['timing_s/merge_val']}"
+                )
+                self.logger.log(data=val_metrics.timing_raw, step=val_metrics.param_version)
+        else:
+            if val_metrics.metrics:
+                self.logger.log(data=val_metrics.metrics, step=val_metrics.param_version)
+                pprint(
+                    f"[FullyAsyncTrainer] parameter version: {val_metrics.param_version} "
+                    f"Validation metrics: {val_metrics.metrics}"
+                )
+        self.logger.log(data=val_metrics.timing_raw, step=val_metrics.param_version)
+
+    async def _validate_process(self):
+        if self.config.async_training.use_trainer_do_validate:
+            print("[FullyAsyncTrainer] _validate_process")
+            from verl.utils.profiler import marked_timer
+
+            timing_raw = {}
+            await self.async_rollout_manager.wake_up()
+            with marked_timer("trainer/validate_time", timing_raw):
+                self.train_val_metrics = self._validate(True)
+            await self.async_rollout_manager.sleep()
+            print(f"[FullyAsyncTrainer] validate timing_raw validate: {timing_raw['trainer/validate_time']}")
+        else:
+            self.train_val_metrics = None
+            print("[FullyAsyncTrainer] _validate_process without async_rollout_manager")

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/megatron_utils.py

@@ -0,0 +1,99 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from megatron.core.distributed import DistributedDataParallel as DDP
+
+
+@torch.no_grad()
+def copy_megatron_model_to_cpu(models):
+    """
+    Copy Megatron model parameters to CPU memory (non-destructive copy).
+    Unlike offload_megatron_model_to_cpu which moves data, this function creates
+    independent copies on CPU while keeping GPU data intact.
+
+    Args:
+        models: List of model chunks (DDP-wrapped or unwrapped)
+
+    Returns:
+        dict: CPU state containing copied parameters and buffers
+    """
+    cpu_state = {}
+
+    for model_idx, model_chunk in enumerate(models):
+        if isinstance(model_chunk, DDP):
+            # Handle DDP-wrapped models
+            model_chunk_all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers]
+            buffer_states = []
+
+            for buffers in model_chunk_all_buffers:
+                buffer_list = []
+                for buffer in buffers:
+                    buffer_state = {}
+
+                    # Copy parameter data to CPU
+                    if buffer.param_data.storage().size() > 0:
+                        buffer_state["param_data"] = buffer.param_data.data.cpu().clone().pin_memory()
+
+                    buffer_list.append(buffer_state)
+                buffer_states.append(buffer_list)
+
+            cpu_state[f"model_chunk_{model_idx}"] = {"buffer_states": buffer_states, "is_ddp": True}
+        else:
+            # Handle non-DDP models (ref module)
+            model_state = {}
+            for name, param in model_chunk.named_parameters():
+                param_state = {"data": param.data.cpu().clone().pin_memory()}
+                model_state[name] = param_state
+
+            cpu_state[f"model_chunk_{model_idx}"] = {"model_state": model_state, "is_ddp": False}
+
+    return cpu_state
+
+
+@torch.no_grad()
+def restore_megatron_model_from_cpu(models, cpu_state):
+    """
+    Restore Megatron model parameters from CPU memory back to GPU.
+
+    Args:
+        models: List of model chunks to restore to
+        cpu_state: CPU state dict returned from copy_megatron_model_to_cpu
+    """
+    for model_idx, model_chunk in enumerate(models):
+        chunk_key = f"model_chunk_{model_idx}"
+        if chunk_key not in cpu_state:
+            continue
+
+        chunk_state = cpu_state[chunk_key]
+
+        if chunk_state["is_ddp"] and isinstance(model_chunk, DDP):
+            # Restore DDP buffers
+            model_chunk_all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers]
+            buffer_states = chunk_state["buffer_states"]
+
+            for buffers, buffer_list in zip(model_chunk_all_buffers, buffer_states, strict=False):
+                for buffer, buffer_state in zip(buffers, buffer_list, strict=False):
+                    # Restore parameter data
+                    if "param_data" in buffer_state:
+                        buffer.param_data.data.copy_(buffer_state["param_data"].to(buffer.param_data.device))
+
+        elif not chunk_state["is_ddp"] and not isinstance(model_chunk, DDP):
+            # Restore non-DDP models
+            model_state = chunk_state["model_state"]
+            for name, param in model_chunk.named_parameters():
+                if name in model_state:
+                    param_state = model_state[name]
+                    param.data.copy_(param_state["data"].to(param.device))

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/megatron_worker.py

@@ -0,0 +1,267 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+# Copyright 2025 NVIDIA Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import time
+
+import torch
+import torch.distributed
+from omegaconf import DictConfig
+
+from verl.experimental.fully_async_policy.base_detach_sync import BaseDetachNcclSync
+from verl.experimental.fully_async_policy.megatron_utils import (
+    copy_megatron_model_to_cpu,
+    restore_megatron_model_from_cpu,
+)
+from verl.single_controller.base.decorator import Dispatch, register
+from verl.utils.device import (
+    get_device_name,
+    get_torch_device,
+)
+from verl.utils.megatron_utils import load_megatron_model_to_gpu, offload_megatron_model_to_cpu, per_tensor_generator
+from verl.workers.megatron_workers import AsyncActorRolloutRefWorker, CriticWorker
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+device_name = get_device_name()
+
+__all__ = ["DetachActorWorker", "DetachAsyncRolloutWorker", "CriticWorker"]
+
+
+class DetachNcclSync(BaseDetachNcclSync, AsyncActorRolloutRefWorker):
+    def __init__(self, config: DictConfig, role: str):
+        BaseDetachNcclSync.__init__(self, config, role)
+
+        AsyncActorRolloutRefWorker.__init__(self, config, role)
+
+    def _get_actor_params(self):
+        pass
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False)
+    def sync_rollout_weights(self, sync_group_name="actor_rollout"):
+        assert (self._is_actor or self._is_rollout) and not self.config.hybrid_engine
+        assert hasattr(self, "_weights_info") and self._weights_info is not None
+        if self._is_actor and self._is_offload_param:
+            load_megatron_model_to_gpu(self.actor_module, False)
+        params_generator = self._get_actor_params_generator() if self._is_actor else None
+        params = {key: tensor for key, tensor in params_generator} if params_generator is not None else None
+
+        rollout_name = self.config.rollout.name
+        inference_model = None
+        if self._is_rollout and (not self._is_actor):
+            if rollout_name == "vllm":
+                inference_model = BaseDetachNcclSync.get_inference_model(self.rollout)
+                from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader
+
+                patch_vllm_moe_model_weight_loader(inference_model)
+            elif rollout_name == "sglang":
+                inference_model = self.rollout._engine
+                if inference_model is None:
+                    print("[sync_rollout_weights] Initialize server adapter engine")
+
+                    async def init_engine():
+                        if hasattr(self.rollout, "_init_server_adapter"):
+                            await self.rollout._init_server_adapter()
+                        else:
+                            print("[sync_rollout_weights] No _init_server_adapter method found")
+                        return self.rollout._engine
+
+                    inference_model = self._run_async_safely(init_engine())
+                    if inference_model is None:
+                        raise RuntimeError(
+                            f"Failed to initialize rollout engine. "
+                            f"rollout type: {type(self.rollout)}, "
+                            f"has _init_server_adapter: {hasattr(self.rollout, '_init_server_adapter')}"
+                        )
+            else:
+                raise NotImplementedError(f"Unknown rollout name: {rollout_name}")
+
+        if rollout_name == "sglang" and self._is_rollout:
+            self._sync_sglang_weights(inference_model, params, sync_group_name)
+        else:
+            self._sync_vllm_weights(inference_model, params, sync_group_name)
+
+        if self._is_actor and self._is_offload_param:
+            offload_megatron_model_to_cpu(self.actor_module)
+            get_torch_device().empty_cache()
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def save_model_to_cpu(self, n):
+        if not hasattr(self, "cpu_saved_models"):
+            self.cpu_saved_models = {}
+        self.cpu_saved_models[n] = copy_megatron_model_to_cpu(self.actor.actor_module)
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def restore_model_from_cpu(self, n):
+        if n in self.cpu_saved_models:
+            restore_megatron_model_from_cpu(self.actor.actor_module, self.cpu_saved_models[n])
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def clear_cpu_model(self, n):
+        if n in self.cpu_saved_models:
+            del self.cpu_saved_models[n]
+
+    def cache_actor_weights_to_cpu(self):
+        self.cpu_named_params = {}
+        if self._is_actor:
+            params_generator = self._get_actor_params_generator()
+            local_rank = torch.distributed.get_rank()
+            world_size = torch.distributed.get_world_size()
+            print(f"cache_actor_weights_to_cpu, local_rank:{local_rank}, world_size:{world_size}")
+            for tensor_idx, (key, tensor) in enumerate(params_generator):
+                if tensor_idx % world_size == local_rank:
+                    self.cpu_named_params[key] = tensor.to("cpu", non_blocking=True)
+            get_torch_device().synchronize()
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False)
+    def sync_rollout_weights_by_checkpoint(self, sync_group_name="actor_rollout"):
+        assert (self._is_actor or self._is_rollout) and not self.config.hybrid_engine
+        assert hasattr(self, "_weights_info") and self._weights_info is not None
+
+        # Load model to GPU
+        load_start_time = time.time()
+        if self._is_actor and self._is_offload_param:
+            load_megatron_model_to_gpu(self.actor_module, False)
+        load_duration = time.time() - load_start_time
+
+        from ray.util.collective import collective
+
+        # Cache actor weights to CPU and measure the time taken
+        cache_start_time = time.time()
+        self.cache_actor_weights_to_cpu()
+        cache_end_time = time.time()
+        cache_duration = cache_end_time - cache_start_time
+
+        # Register the cached weights into the checkpoint engine
+        self.checkpoint_engine.register_checkpoint(self._weights_info, self.cpu_named_params)
+        register_end_time = time.time()
+        register_duration = register_end_time - cache_end_time
+        self.cpu_named_params = {}
+
+        collective.barrier(group_name=sync_group_name)
+        update_start_time = time.time()
+
+        rollout_name = self.config.rollout.name
+        inference_model = None
+        if self._is_rollout and (not self._is_actor):
+            if rollout_name == "vllm":
+                inference_model = BaseDetachNcclSync.get_inference_model(self.rollout)
+                from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader
+
+                patch_vllm_moe_model_weight_loader(inference_model)
+            elif rollout_name == "sglang":
+                inference_model = self.rollout._engine
+                # For ServerAdapter, _engine might be None and needs async initialization
+                if inference_model is None:
+                    # Initialize the server adapter engine
+                    print("[sync_rollout_weights] Initialize server adapter engine")
+
+                    async def init_engine():
+                        if hasattr(self.rollout, "_init_server_adapter"):
+                            await self.rollout._init_server_adapter()
+                        else:
+                            print("[sync_rollout_weights] No _init_server_adapter method found")
+                        return self.rollout._engine
+
+                    inference_model = self._run_async_safely(init_engine())
+                    if inference_model is None:
+                        raise RuntimeError(
+                            f"Failed to initialize rollout engine. "
+                            f"rollout type: {type(self.rollout)}, "
+                            f"has _init_server_adapter: {hasattr(self.rollout, '_init_server_adapter')}"
+                        )
+            else:
+                raise NotImplementedError(f"Unknown rollout name: {rollout_name}")
+        # Update the checkpoint with the inference model and broadcast weights
+        self.checkpoint_engine.update_checkpoint(
+            inference_model=inference_model,
+            group_name=sync_group_name,
+            overlap_broadcast_and_consume=self.config.checkpoint_engine.overlap_broadcast_and_consume,
+        )
+
+        update_end_time = time.time()
+        update_duration = update_end_time - update_start_time
+
+        collective.barrier(group_name=sync_group_name)
+        offload_start_time = time.time()
+        if self._is_actor and self._is_offload_param:
+            offload_megatron_model_to_cpu(self.actor_module)
+        offload_duration = time.time() - offload_start_time
+
+        print(
+            f"sync_rollout_weights_by_checkpoint finish!, rank:{torch.distributed.get_rank()},"
+            f" is_actor:{self._is_actor}, is_rollout:{self._is_rollout},"
+            f" total cost:{update_end_time - cache_start_time} seconds, while cache cost {cache_duration} seconds, "
+            f" register cost {register_duration} seconds, update cost {update_duration} seconds"
+        )
+
+        if self._is_actor and self._is_offload_param:
+            print(
+                f"sync_rollout_weights_by_checkpoint load model to gpu cost {load_duration} seconds,"
+                f" offload model to cpu cost {offload_duration} seconds"
+            )
+
+
+class DetachActorWorker(DetachNcclSync):
+    def __init__(self, config: DictConfig, role: str):
+        print("[DetachAsyncRolloutWorker] Initializing via DetachNcclSync...")
+        DetachNcclSync.__init__(self, config, role)
+
+    def _get_actor_params_generator(self):
+        assert self._is_actor
+        if self.bridge is not None:
+            generator = self.bridge.export_weights(self.actor.actor_module)
+        else:
+            generator = per_tensor_generator(
+                self.actor.actor_module,
+                self.actor_model_config,
+                self.weight_converter,
+                self.tf_config,
+                self.layer_name_mapping,
+            )
+
+        return generator
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def get_actor_weights_info(self):
+        assert self._is_actor
+        if hasattr(self, "_weights_info"):
+            return self._weights_info
+        if self._is_offload_param:
+            load_megatron_model_to_gpu(self.actor_module, False)
+        params_generator = self._get_actor_params_generator()
+        ret = []
+        for key, tensor in params_generator:
+            ret.append((key, tensor.size(), tensor.dtype))
+
+        self._weights_info = ret
+        # Here, we only call this function at the beginning,
+        # and immediately afterwards we call sync_rollout_weights.
+        # So we no longer call offload in this.
+        return ret
+
+
+class DetachAsyncRolloutWorker(DetachNcclSync):
+    def __init__(self, config: DictConfig, role: str):
+        print(f"[DetachAsyncRolloutWorker] {DetachAsyncRolloutWorker.__mro__}")
+        DetachNcclSync.__init__(self, config, role)
+
+    @register(dispatch_mode=Dispatch.ONE_TO_ALL)
+    def set_actor_weights_info(self, weights_info):
+        assert self._is_rollout
+        self._weights_info = weights_info

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/message_queue.py

@@ -0,0 +1,265 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import logging
+from collections import deque
+from typing import Any
+
+import ray
+from omegaconf import DictConfig
+
+logger = logging.getLogger(__name__)
+
+
+@ray.remote(num_cpus=2, max_concurrency=20)
+class MessageQueue:
+    """
+    Simplified Ray-based asynchronous message queue for communication between Rollouter and Trainer
+    """
+
+    def __init__(self, config: DictConfig, max_queue_size: int = 1000):
+        self.config = config
+        if max_queue_size is None:
+            raise ValueError(f"max_queue_size cannot be None, got: {max_queue_size}")
+        self.max_queue_size = int(max_queue_size)
+        self.queue = deque(maxlen=self.max_queue_size)
+        self.current_param_version = 0
+
+        self.val_queue = deque()
+
+        try:
+            if hasattr(config, "async_training") and config.async_training is not None:
+                self.staleness_threshold = getattr(config.async_training, "staleness_threshold", 3)
+            else:
+                self.staleness_threshold = 3
+        except (AttributeError, RecursionError):
+            self.staleness_threshold = 3
+
+        # Asyncio for message handling
+        self.running = True
+
+        # async safe
+        self._lock = asyncio.Lock()
+        self._consumer_condition = asyncio.Condition(self._lock)
+
+        # statistic message
+        self.total_produced = 0
+        self.total_consumed = 0
+        self.dropped_samples = 0
+
+        print(
+            f"[MessageQueue] initialized with max_queue_size={max_queue_size},"
+            f"staleness_threshold={self.staleness_threshold}"
+        )
+
+    async def put_sample(self, sample: Any, param_version: int) -> bool:
+        """
+        Put a batch sample into the queue
+
+        Args:
+            sample: Sample data
+            param_version: Parameter version number
+
+        Returns:
+            bool: Whether the sample was successfully put into the queue
+        """
+        async with self._lock:
+            # If queue is full, remove the oldest sample (rarely happens)
+            is_drop = False
+            if len(self.queue) >= self.max_queue_size:
+                self.queue.popleft()
+                self.dropped_samples += 1
+                is_drop = True
+                logger.warning("Queue full, dropped sample")
+            self.queue.append(sample)
+            self.total_produced += 1
+
+            # Notify waiting consumers
+            self._consumer_condition.notify_all()
+
+            if self.total_produced % 100 == 0:
+                print(f"MessageQueue stats: produced={self.total_produced}, queue_size={len(self.queue)}")
+            if is_drop:
+                return False
+            return True
+
+    async def get_sample(self) -> Any | None:
+        """
+        Get a single sample from the queue, wait until one is available
+
+        Returns:
+            Any: Single sample data or None if queue is closed
+        """
+        async with self._lock:
+            while len(self.queue) == 0 and self.running:
+                await self._consumer_condition.wait()
+
+            # If queue is closed and empty, return None
+            if not self.running and len(self.queue) == 0:
+                return None
+
+            # Get one sample
+            data = self.queue.popleft()
+            self.total_consumed += 1
+            return data, len(self.queue)
+
+    async def update_param_version(self, version: int):
+        """Update current parameter version"""
+        async with self._lock:
+            old_version = self.current_param_version
+            self.current_param_version = version
+            print(f"Parameter version updated from {old_version} to {version}")
+
+    async def get_queue_size(self) -> int:
+        """Get current queue length"""
+        async with self._lock:
+            return len(self.queue)
+
+    async def get_statistics(self) -> dict[str, Any]:
+        """Get queue statistics"""
+        async with self._lock:
+            return {
+                "queue_size": len(self.queue),
+                "total_produced": self.total_produced,
+                "total_consumed": self.total_consumed,
+                "dropped_samples": self.dropped_samples,
+                "current_param_version": self.current_param_version,
+                "staleness_threshold": self.staleness_threshold,
+                "max_queue_size": self.max_queue_size,
+            }
+
+    async def clear_queue(self):
+        """Clear the queue"""
+        async with self._lock:
+            cleared_count = len(self.queue)
+            self.queue.clear()
+            logger.info(f"Cleared {cleared_count} samples from queue")
+
+    async def shutdown(self):
+        """Shutdown the message queue"""
+        async with self._lock:
+            self.running = False
+            # Notify all waiting coroutines so they can exit
+            self._consumer_condition.notify_all()
+        logger.info("MessageQueue shutdown")
+
+    async def get_memory_usage(self) -> dict:
+        """Get memory usage statistics"""
+        async with self._lock:
+            # Estimate memory usage of samples in queue
+            import sys
+
+            total_size = 0
+            sample_count = len(self.queue)
+
+            if sample_count > 0:
+                # Estimate size of a single sample (simplified estimation)
+                sample = list(self.queue)[0]
+                try:
+                    sample_size = sys.getsizeof(sample)
+                    # Since we now store RolloutSample directly, estimate based on its components
+                    if hasattr(sample, "original_batch_dict") and sample.original_batch_dict:
+                        # Estimate batch data size
+                        batch_data = sample.original_batch_dict.get("batch", {})
+                        sample_size += len(batch_data) * 1000  # Roughly estimate 1KB per batch entry
+                    if hasattr(sample, "agent_loop_output"):
+                        # Estimate AgentLoopOutput size
+                        sample_size += 5000  # Roughly estimate 5KB for AgentLoopOutput
+                    total_size = sample_size * sample_count
+                except Exception:
+                    total_size = sample_count * 15000  # Roughly estimate 15KB per RolloutSample
+
+            return {
+                "queue_samples": sample_count,
+                "estimated_memory_bytes": total_size,
+                "estimated_memory_mb": total_size / (1024 * 1024),
+            }
+
+    async def put_validate(self, data):
+        async with self._lock:
+            self.val_queue.append(data)
+
+    async def get_validate(self):
+        async with self._lock:
+            if self.val_queue:
+                return self.val_queue.popleft()
+            else:
+                return None
+
+
+class MessageQueueClient:
+    """Asyncio-compatible MessageQueue client for communicating with MessageQueue Actor"""
+
+    def __init__(self, queue_actor: Any):
+        self.queue_actor = queue_actor
+
+    async def put_sample(self, sample: Any, param_version: int) -> bool:
+        """Put batch into queue (async)"""
+        future = self.queue_actor.put_sample.remote(sample, param_version)
+        return await asyncio.wrap_future(future.future())
+
+    async def put_validate(self, data: Any) -> bool:
+        future = self.queue_actor.put_validate.remote(data)
+        return await asyncio.wrap_future(future.future())
+
+    def get_validate_sync(self) -> Any | None:
+        return ray.get(self.queue_actor.get_validate.remote())
+
+    async def get_sample(self) -> Any | None:
+        """Get single sample from queue, wait until one is available (async)"""
+        future = self.queue_actor.get_sample.remote()
+        return await asyncio.wrap_future(future.future())
+
+    async def get_queue_size(self) -> int:
+        """Get queue size (async)"""
+        future = self.queue_actor.get_queue_size.remote()
+        return await asyncio.wrap_future(future.future())
+
+    async def get_statistics(self) -> dict[str, Any]:
+        """Get statistics (async)"""
+        future = self.queue_actor.get_statistics.remote()
+        return await asyncio.wrap_future(future.future())
+
+    async def clear_queue(self):
+        """Clear queue (async)"""
+        future = self.queue_actor.clear_queue.remote()
+        await asyncio.wrap_future(future.future())
+
+    async def shutdown(self):
+        """Shutdown queue (async)"""
+        future = self.queue_actor.shutdown.remote()
+        await asyncio.wrap_future(future.future())
+
+    async def get_memory_usage(self) -> dict:
+        """Get memory usage statistics (async)"""
+        future = self.queue_actor.get_memory_usage.remote()
+        return await asyncio.wrap_future(future.future())
+
+    # Synchronous version of the method (deprecated)
+    def put_sample_sync(self, sample: Any, param_version: int) -> bool:
+        """Put batch into queue (sync - deprecated, use put_sample instead)"""
+        return ray.get(self.queue_actor.put_sample.remote(sample, param_version))
+
+    def get_sample_sync(self) -> Any | None:
+        """Get single sample from queue (sync - deprecated, use get_sample instead)"""
+        return ray.get(self.queue_actor.get_sample.remote())
+
+    def get_statistics_sync(self) -> dict[str, Any]:
+        """Get statistics (sync - deprecated, use get_statistics instead)"""
+        return ray.get(self.queue_actor.get_statistics.remote())
+
+    def update_param_version_sync(self, version: int):
+        """Update parameter version (async)"""
+        return ray.get(self.queue_actor.update_param_version.remote(version))

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/param_sync.py

@@ -0,0 +1,173 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+
+import ray
+from ray.util.collective import collective
+
+from verl.utils.device import get_nccl_backend
+
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class ParameterSynchronizer:
+    """
+    Unified parameter synchronizer, responsible for synchronizing model parameters between actor and rollout
+    Based on the mature synchronization mode implementation of one_step_off_policy
+    Merges the functions of the original multiple synchronizer classes
+    """
+
+    def __init__(self, config, trainer, rollouter, mq):
+        self.config = config
+        self.trainer = trainer
+        self.rollouter = rollouter
+        self.mq_client = mq
+        self.actor_wg = ray.get(trainer.get_actor_wg.remote())
+        self.rollout_wg = ray.get(rollouter.get_rollout_wg.remote())
+
+        # Basic attributes
+        self.weights_info = None
+        self.sync_group_initialized = False
+        self.sync_group_name = "actor_rollout"
+        self.wait_last_update = None
+        self.wait_last_resume = None
+        self.validate_task = None
+
+        # Statistics
+        self.current_version = 0
+
+        self._init_weights_info()
+        self._init_sync_group()
+
+        if self.config.async_training.checkpoint_engine.enable:
+            self._init_actor_rollout_checkpoint_engine()
+
+    def get_current_param_version(self) -> int:
+        """Get current parameter version number"""
+        return self.current_version
+
+    def get_weights_info(self):
+        """Get weights info"""
+        return self.weights_info
+
+    def _init_weights_info(self):
+        self.weights_info = self.actor_wg.get_actor_weights_info()[0]
+        self.rollout_wg.set_actor_weights_info(self.weights_info)
+
+    def _init_sync_group(self):
+        print("[ParameterSynchronizer] Initializing parameter synchronization group...")
+        actor_rollout_workers = self.actor_wg.workers + self.rollout_wg.workers
+        n_workers = len(self.actor_wg.workers + self.rollout_wg.workers)
+        if self.config.trainer.device == "npu":
+            master_address = ray.get(self.actor_wg.workers[0]._get_node_ip.remote()).strip("[]")
+            master_port = ray.get(self.actor_wg.workers[0]._get_free_port.remote())
+            self.actor_wg.create_weight_sync_group(
+                master_address,
+                master_port,
+                0,
+                n_workers,
+            )
+            ray.get(
+                self.rollout_wg.create_weight_sync_group(
+                    master_address,
+                    master_port,
+                    len(self.actor_wg.workers),
+                    n_workers,
+                )
+            )
+        else:
+            collective.create_collective_group(
+                actor_rollout_workers,
+                n_workers,
+                list(range(0, n_workers)),
+                backend=get_nccl_backend(),
+                group_name=self.sync_group_name,
+            )
+
+    def _init_actor_rollout_checkpoint_engine(self):
+        ray.get(
+            self.actor_wg.init_checkpoint_engine(
+                rank_offset=0,
+                actor_num=len(self.actor_wg.workers),
+                rollout_num=len(self.rollout_wg.workers),
+            )
+        )
+        ray.get(
+            self.rollout_wg.init_checkpoint_engine(
+                rank_offset=len(self.actor_wg.workers),
+                actor_num=len(self.actor_wg.workers),
+                rollout_num=len(self.rollout_wg.workers),
+            )
+        )
+
+    def sync_weights(self, version, validate=False, global_steps=0, use_trainer_do_validate=False):
+        """Sync weights between trainer and rollouter, and update parameter version"""
+        start_time = time.time()
+
+        self.current_version = version
+        ray.get(self.rollouter.pause.remote())
+
+        print(f"[ParameterSynchronizer] rollout paused. cost {time.time() - start_time:.2f} seconds")
+        # Update MQ version
+        self.mq_client.update_param_version_sync(version)
+
+        pause_time = time.time()
+
+        # sync weights
+        # For sglang, always use sync_rollout_weights instead of sync_rollout_weights_by_checkpoint
+        rollout_name = getattr(self.config.actor_rollout_ref.rollout, "name", None)
+        use_checkpoint_engine = self.config.async_training.checkpoint_engine.enable and rollout_name != "sglang"
+
+        if use_checkpoint_engine:
+            self.actor_wg.sync_rollout_weights_by_checkpoint(self.sync_group_name)
+            ray.get(self.rollout_wg.sync_rollout_weights_by_checkpoint(self.sync_group_name))
+        else:
+            self.actor_wg.sync_rollout_weights(self.sync_group_name)
+            ray.get(self.rollout_wg.sync_rollout_weights(self.sync_group_name))
+        end_time = time.time()
+        print(
+            f"[ParameterSynchronizer] sync_weights success. cost {end_time - start_time:.2f} seconds, "
+            f"pause:{pause_time - start_time:.2f}s, sync:{end_time - pause_time:.2f}s"
+        )
+        # async train do validate
+        print(f"[ParameterSynchronizer] validate: {validate}, use_trainer_do_validate: {use_trainer_do_validate}")
+        if validate and use_trainer_do_validate:
+            print("[ParameterSynchronizer] use trainer to do validate")
+            self.validate_task = self.trainer._validate_process.remote()
+        else:
+            self.validate_task = None
+        # Async Update rollout version & validation
+        self.wait_last_update = self.rollouter.update_param_version.remote(
+            version, validate, global_steps, use_trainer_do_validate
+        )
+        self.wait_last_resume = self.rollouter.resume.remote(self.wait_last_update)
+
+    def wait_last_valid(self):
+        print("[ParameterSynchronizer] Waiting last sync and validate...")
+        start_time = time.time()
+        if self.wait_last_update:
+            ray.get(self.wait_last_update)
+        if self.wait_last_resume:
+            ray.get(self.wait_last_resume)
+        if self.validate_task:
+            ray.get(self.validate_task)
+        print(f"[ParameterSynchronizer] Wait last validate cost: {time.time() - start_time:.2f} seconds")
+
+    def rollouter_save_checkpoint(self, local_global_step_folder: str):
+        """Trigger rollout to save checkpoint(dataloader)"""
+        print(f"[ParameterSynchronizer] Triggering checkpoint save at {local_global_step_folder} ...")
+        return ray.get(self.rollouter.save_checkpoint.remote(local_global_step_folder))

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/ray_trainer.py

@@ -0,0 +1,538 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+# Copyright 2023-2024 SGLang Team
+# Copyright 2025 ModelBest Inc. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+PPO Trainer with Ray-based single controller.
+This trainer supports model-agonistic model initialization with huggingface
+"""
+
+import uuid
+from copy import deepcopy
+from pprint import pprint
+
+import numpy as np
+import ray
+import torch
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from verl import DataProto
+from verl.experimental.dataset.sampler import AbstractCurriculumSampler
+from verl.single_controller.ray import RayClassWithInitArgs
+from verl.single_controller.ray.base import create_colocated_worker_cls
+from verl.trainer.ppo.core_algos import AdvantageEstimator, agg_loss
+from verl.trainer.ppo.metric_utils import (
+    compute_data_metrics,
+    compute_throughout_metrics,
+    compute_timing_metrics,
+)
+from verl.trainer.ppo.ray_trainer import RayPPOTrainer, apply_kl_penalty, compute_advantage, compute_response_mask
+from verl.trainer.ppo.reward import compute_reward, compute_reward_async
+from verl.trainer.ppo.utils import Role
+from verl.utils.config import omega_conf_to_dataclass
+from verl.utils.debug import marked_timer
+from verl.utils.metric import (
+    reduce_metrics,
+)
+from verl.utils.rollout_skip import RolloutSkip
+
+
+class FullyAsyncRayPPOTrainer(RayPPOTrainer):
+    def init_workers(self):
+        """Initialize distributed training workers using Ray backend.
+
+        Creates:
+        1. Ray resource pools from configuration
+        2. Worker groups for each role (actor, critic, etc.)
+        """
+        self._init_resource_pools()
+        self._create_worker_classes()
+        self._init_worker_groups()
+        self._init_models()
+        self._init_async_rollout_manager()
+
+    def _init_resource_pools(self):
+        self.resource_pool_manager.create_resource_pool()
+
+        self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()}
+
+    def _create_worker_classes(self):
+        self._create_actor_rollout_classes()
+        self._create_critic_class()
+        self._create_reference_policy_class()
+        self._create_reward_model_class()
+
+    def _create_actor_rollout_classes(self):
+        raise NotImplementedError
+
+    def _create_critic_class(self):
+        # create critic
+        if self.use_critic:
+            resource_pool = self.resource_pool_manager.get_resource_pool(Role.Critic)
+            critic_cfg = omega_conf_to_dataclass(self.config.critic)
+            critic_cls = RayClassWithInitArgs(cls=self.role_worker_mapping[Role.Critic], config=critic_cfg)
+            self.resource_pool_to_cls[resource_pool][str(Role.Critic)] = critic_cls
+
+    def _create_reference_policy_class(self):
+        # create reference policy if needed
+        if self.use_reference_policy:
+            resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy)
+            ref_policy_cls = RayClassWithInitArgs(
+                self.role_worker_mapping[Role.RefPolicy],
+                config=self.config.actor_rollout_ref,
+                role=str(Role.RefPolicy),
+                # profile_option=self.config.trainer.npu_profile.options,
+            )
+            self.resource_pool_to_cls[resource_pool][str(Role.RefPolicy)] = ref_policy_cls
+
+    def _create_reward_model_class(self):
+        # create a reward model if reward_fn is None
+        if self.use_rm:
+            # we create a RM here
+            resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel)
+            rm_cls = RayClassWithInitArgs(self.role_worker_mapping[Role.RewardModel], config=self.config.reward_model)
+            self.resource_pool_to_cls[resource_pool][str(Role.RewardModel)] = rm_cls
+
+    def _init_worker_groups(self):
+        # initialize WorkerGroup
+        # NOTE: if you want to use a different resource pool for each role, which can support different parallel size,
+        # you should not use `create_colocated_worker_cls`.
+        # Instead, directly pass different resource pool to different worker groups.
+        # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information.
+        all_wg = {}
+        wg_kwargs = {}  # Setting up kwargs for RayWorkerGroup
+        if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None:
+            wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout
+        if OmegaConf.select(self.config.global_profiler, "steps") is not None:
+            wg_kwargs["profile_steps"] = OmegaConf.select(self.config.global_profiler, "steps")
+            # Only require nsight worker options when tool is nsys
+            if OmegaConf.select(self.config.global_profiler, "tool") == "nsys":
+                assert (
+                    OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options")
+                    is not None
+                ), "worker_nsight_options must be set when using nsys with profile_steps"
+                wg_kwargs["worker_nsight_options"] = OmegaConf.to_container(
+                    OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options")
+                )
+        wg_kwargs["device_name"] = self.device_name
+
+        for resource_pool, class_dict in self.resource_pool_to_cls.items():
+            worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict)
+            wg_dict = self.ray_worker_group_cls(
+                resource_pool=resource_pool,
+                ray_cls_with_init=worker_dict_cls,
+                **wg_kwargs,
+            )
+            spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys())
+            all_wg.update(spawn_wg)
+        self.all_wg = all_wg
+
+    def _init_models(self):
+        if self.use_critic:
+            self.critic_wg = self.all_wg[str(Role.Critic)]
+            self.critic_wg.init_model()
+
+        if self.use_reference_policy and not self.ref_in_actor:
+            self.ref_policy_wg = self.all_wg[str(Role.RefPolicy)]
+            self.ref_policy_wg.init_model()
+
+        if self.use_rm:
+            self.rm_wg = self.all_wg[str(Role.RewardModel)]
+            self.rm_wg.init_model()
+
+        # we should create rollout at the end so that vllm can have a better estimation of kv cache memory
+        self.actor_rollout_wg = self.all_wg[str(Role.ActorRollout)]
+        self.actor_rollout_wg.init_model()
+
+    def _init_async_rollout_manager(self):
+        pass
+
+    def fit(self):
+        """
+        The training loop of PPO.
+        The driver process only need to call the compute functions of the worker group through RPC
+        to construct the PPO dataflow.
+        The light-weight advantage computation is done on the driver process.
+        """
+        from omegaconf import OmegaConf
+
+        from verl.utils.tracking import Tracking
+
+        logger = Tracking(
+            project_name=self.config.trainer.project_name,
+            experiment_name=self.config.trainer.experiment_name,
+            default_backend=self.config.trainer.logger,
+            config=OmegaConf.to_container(self.config, resolve=True),
+        )
+
+        self.global_steps = 0
+
+        # load checkpoint before doing anything
+        self._load_checkpoint()
+
+        # perform validation before training
+        # currently, we only support validation using the reward_function.
+        if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True):
+            val_metrics = self._validate()
+            assert val_metrics, f"{val_metrics=}"
+            pprint(f"Initial validation metrics: {val_metrics}")
+            logger.log(data=val_metrics, step=self.global_steps)
+            if self.config.trainer.get("val_only", False):
+                return
+
+        if self.config.actor_rollout_ref.rollout.get("skip_rollout", False):
+            rollout_skip = RolloutSkip(self.config, self.actor_rollout_wg)
+            rollout_skip.wrap_generate_sequences()
+
+        # add tqdm
+        progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress")
+
+        # we start from step 1
+        self.global_steps += 1
+        last_val_metrics = None
+        self.max_steps_duration = 0
+
+        prev_step_profile = False
+        curr_step_profile = (
+            self.global_steps in self.config.global_profiler.steps
+            if self.config.global_profiler.steps is not None
+            else False
+        )
+        next_step_profile = False
+
+        for epoch in range(self.config.trainer.total_epochs):
+            for batch_dict in self.train_dataloader:
+                metrics = {}
+                timing_raw = {}
+
+                with marked_timer("start_profile", timing_raw):
+                    self._start_profiling(
+                        not prev_step_profile and curr_step_profile
+                        if self.config.global_profiler.profile_continuous_steps
+                        else curr_step_profile
+                    )
+
+                batch, gen_batch = self._prepare_generate_batch(batch_dict)
+
+                is_last_step = self.global_steps >= self.total_training_steps
+
+                with marked_timer("step", timing_raw):
+                    # generate a batch
+                    with marked_timer("gen", timing_raw, color="red"):
+                        if not self.async_rollout_mode:
+                            gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch)
+                        else:
+                            gen_batch_output = self.async_rollout_manager.generate_sequences(gen_batch)
+                        timing_raw.update(gen_batch_output.meta_info["timing"])
+                        gen_batch_output.meta_info.pop("timing", None)
+
+                    if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX:
+                        if self.reward_fn is None:
+                            raise ValueError("A reward_fn is required for REMAX advantage estimation.")
+
+                        with marked_timer("gen_max", timing_raw, color="purple"):
+                            gen_baseline_batch = deepcopy(gen_batch)
+                            gen_baseline_batch.meta_info["do_sample"] = False
+                            if not self.async_rollout_mode:
+                                gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch)
+                            else:
+                                gen_baseline_output = self.async_rollout_manager.generate_sequences(gen_baseline_batch)
+                            batch = batch.union(gen_baseline_output)
+                            reward_baseline_tensor = self.reward_fn(batch)
+                            reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1)
+
+                            batch.pop(batch_keys=list(gen_baseline_output.batch.keys()))
+
+                            batch.batch["reward_baselines"] = reward_baseline_tensor
+
+                            del gen_baseline_batch, gen_baseline_output
+
+                    batch = self._post_generate_batch(batch, gen_batch_output, metrics)
+                    batch, reward_extra_infos_dict = self._process_batch_common(batch, metrics, timing_raw)
+                    self._log_rollout(batch, reward_extra_infos_dict, timing_raw)
+
+                last_val_metrics = self._validate_metrics(is_last_step, last_val_metrics, metrics, timing_raw)
+                self._check_save_checkpoint(is_last_step, timing_raw)
+
+                with marked_timer("stop_profile", timing_raw):
+                    next_step_profile = (
+                        self.global_steps + 1 in self.config.global_profiler.steps
+                        if self.config.global_profiler.steps is not None
+                        else False
+                    )
+                    self._stop_profiling(
+                        curr_step_profile and not next_step_profile
+                        if self.config.global_profiler.profile_continuous_steps
+                        else curr_step_profile
+                    )
+                    prev_step_profile = curr_step_profile
+                    curr_step_profile = next_step_profile
+
+                self._collect_metrics(batch, epoch, metrics, timing_raw)
+                self._post_batch_processing(batch)
+
+                # TODO: make a canonical logger that supports various backend
+                logger.log(data=metrics, step=self.global_steps)
+
+                progress_bar.update(1)
+                self.global_steps += 1
+
+                if (
+                    hasattr(self.config.actor_rollout_ref.actor, "profiler")
+                    and self.config.actor_rollout_ref.actor.profiler.tool == "torch_memory"
+                ):
+                    self.actor_rollout_wg.dump_memory_snapshot(
+                        tag=f"post_update_step{self.global_steps}", sub_dir=f"step{self.global_steps}"
+                    )
+
+                if is_last_step:
+                    pprint(f"Final validation metrics: {last_val_metrics}")
+                    progress_bar.close()
+                    return
+
+    def _prepare_generate_batch(self, batch_dict):
+        batch: DataProto = DataProto.from_single_dict(batch_dict)
+
+        # add uid to batch
+        batch.non_tensor_batch["uid"] = np.array([str(uuid.uuid4()) for _ in range(len(batch.batch))], dtype=object)
+
+        gen_batch = self._get_gen_batch(batch)
+
+        # pass global_steps to trace
+        gen_batch.meta_info["global_steps"] = self.global_steps
+        gen_batch = gen_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True)
+        return batch, gen_batch
+
+    def _post_generate_batch(self, batch, gen_batch_output, metrics):
+        # repeat to align with repeated responses in rollout
+        batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True)
+        batch = batch.union(gen_batch_output)
+
+        if "response_mask" not in batch.batch.keys():
+            batch.batch["response_mask"] = compute_response_mask(batch)
+        # Balance the number of valid tokens across DP ranks.
+        # NOTE: This usually changes the order of data in the `batch`,
+        # which won't affect the advantage calculation (since it's based on uid),
+        # but might affect the loss calculation (due to the change of mini-batching).
+        # TODO: Decouple the DP balancing and mini-batching.
+        if self.config.trainer.balance_batch:
+            self._balance_batch(batch, metrics=metrics)
+
+        # compute global_valid tokens
+        batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist()
+
+        return batch
+
+    def _process_batch_common(self, batch, metrics, timing_raw, local_trigger_step=None):
+        with marked_timer("reward", timing_raw, color="yellow"):
+            # compute reward model score
+            if self.use_rm:
+                reward_tensor = self.rm_wg.compute_rm_score(batch)
+                batch = batch.union(reward_tensor)
+
+            if self.config.reward_model.launch_reward_fn_async:
+                future_reward = compute_reward_async.remote(data=batch, reward_fn=self.reward_fn)
+            else:
+                reward_tensor, reward_extra_infos_dict = compute_reward(batch, self.reward_fn)
+
+        with marked_timer("old_log_prob", timing_raw, color="blue"):
+
+            def compute_old_log_prob(batch):
+                old_log_prob = self.actor_rollout_wg.compute_log_prob(batch)
+                entropys = old_log_prob.batch["entropys"]
+                response_masks = batch.batch["response_mask"]
+                actor_config = self.config.actor_rollout_ref.actor
+                entropy_agg = agg_loss(
+                    loss_mat=entropys,
+                    loss_mask=response_masks,
+                    loss_agg_mode=actor_config.loss_agg_mode,
+                    loss_scale_factor=actor_config.loss_scale_factor,
+                )
+                old_log_prob_metrics = {"actor/entropy": entropy_agg.detach().item()}
+                metrics.update(old_log_prob_metrics)
+                old_log_prob.batch.pop("entropys")
+                batch = batch.union(old_log_prob)
+                if "rollout_log_probs" in batch.batch.keys():
+                    # TODO: we may want to add diff of probs too.
+                    from verl.utils.debug.metrics import calculate_debug_metrics
+
+                    metrics.update(calculate_debug_metrics(batch))
+                return batch
+
+            async_training = self.config.get("async_training", None)
+            if async_training and async_training.use_rollout_log_probs:
+                # If local_triger_step == 1, load the training engine's parameters to the CPU
+                #  and save a copy for subsequent MIS use.
+                # If local_trigger_step == 2, 3, ..., restore the parameters of version 1 to calculate the old_log_prob,
+                # then restore the parameters of the current version.
+                if local_trigger_step == 1:
+                    self.actor_rollout_wg.save_model_to_cpu(1)
+                    batch = compute_old_log_prob(batch)
+                elif local_trigger_step is not None:
+                    self.actor_rollout_wg.save_model_to_cpu(local_trigger_step)
+                    self.actor_rollout_wg.restore_model_from_cpu(1)
+                    batch = compute_old_log_prob(batch)
+                    self.actor_rollout_wg.restore_model_from_cpu(local_trigger_step)
+                    self.actor_rollout_wg.clear_cpu_model(local_trigger_step)
+                else:
+                    batch.batch["old_log_probs"] = batch.batch["rollout_log_probs"]
+                    batch.meta_info["temperature"] = self.config.actor_rollout_ref.rollout.temperature
+
+            else:
+                batch = compute_old_log_prob(batch)
+
+        if self.use_reference_policy:
+            # compute reference log_prob
+            with marked_timer("ref", timing_raw, color="olive"):
+                if not self.ref_in_actor:
+                    ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch)
+                else:
+                    ref_log_prob = self.actor_rollout_wg.compute_ref_log_prob(batch)
+                batch = batch.union(ref_log_prob)
+
+        # compute values
+        if self.use_critic:
+            with marked_timer("values", timing_raw, color="cyan"):
+                values = self.critic_wg.compute_values(batch)
+                batch = batch.union(values)
+
+        with marked_timer("adv", timing_raw, color="brown"):
+            # we combine with rule-based rm
+            reward_extra_infos_dict: dict[str, list]
+            if self.config.reward_model.launch_reward_fn_async:
+                reward_tensor, reward_extra_infos_dict = ray.get(future_reward)
+            batch.batch["token_level_scores"] = reward_tensor
+
+            if reward_extra_infos_dict:
+                batch.non_tensor_batch.update({k: np.array(v) for k, v in reward_extra_infos_dict.items()})
+
+            # compute rewards. apply_kl_penalty if available
+            if self.config.algorithm.use_kl_in_reward:
+                batch, kl_metrics = apply_kl_penalty(
+                    batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty
+                )
+                metrics.update(kl_metrics)
+            else:
+                batch.batch["token_level_rewards"] = batch.batch["token_level_scores"]
+
+            # Compute rollout correction weights centrally (once per batch)
+            # This corrects for off-policy issues (policy mismatch, model staleness, etc.)
+            # Also computes off-policy diagnostic metrics (KL, PPL, etc.)
+            from verl.trainer.ppo.rollout_corr_helper import compute_rollout_correction_and_add_to_batch
+
+            rollout_corr_config = self.config.algorithm.get("rollout_correction", None)
+            if rollout_corr_config is not None and "rollout_log_probs" in batch.batch:
+                batch, is_metrics = compute_rollout_correction_and_add_to_batch(batch, rollout_corr_config)
+                # IS and off-policy metrics already have rollout_corr/ prefix
+                metrics.update(is_metrics)
+
+            # compute advantages, executed on the driver process
+            norm_adv_by_std_in_grpo = self.config.algorithm.get(
+                "norm_adv_by_std_in_grpo", True
+            )  # GRPO adv normalization factor
+
+            batch = compute_advantage(
+                batch,
+                adv_estimator=self.config.algorithm.adv_estimator,
+                gamma=self.config.algorithm.gamma,
+                lam=self.config.algorithm.lam,
+                num_repeat=self.config.actor_rollout_ref.rollout.n,
+                norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo,
+                config=self.config.algorithm,
+            )
+
+        # update critic
+        if self.use_critic:
+            with marked_timer("update_critic", timing_raw, color="pink"):
+                critic_output = self.critic_wg.update_critic(batch)
+            critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"])
+            metrics.update(critic_output_metrics)
+
+        # implement critic warmup
+        if self.config.trainer.critic_warmup <= self.global_steps:
+            # update actor
+            with marked_timer("update_actor", timing_raw, color="red"):
+                batch.meta_info["multi_turn"] = self.config.actor_rollout_ref.rollout.multi_turn.enable
+                actor_output = self.actor_rollout_wg.update_actor(batch)
+            actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"])
+            metrics.update(actor_output_metrics)
+        return batch, reward_extra_infos_dict
+
+    def _log_rollout(self, batch, reward_extra_infos_dict, timing_raw):
+        # Log rollout generations if enabled
+        rollout_data_dir = self.config.trainer.get("rollout_data_dir", None)
+        if rollout_data_dir:
+            with marked_timer("dump_rollout_generations", timing_raw, color="green"):
+                inputs = self.tokenizer.batch_decode(batch.batch["prompts"], skip_special_tokens=True)
+                outputs = self.tokenizer.batch_decode(batch.batch["responses"], skip_special_tokens=True)
+                scores = batch.batch["token_level_scores"].sum(-1).cpu().tolist()
+                sample_gts = [item.non_tensor_batch.get("reward_model", {}).get("ground_truth", None) for item in batch]
+
+                if "request_id" in batch.non_tensor_batch:
+                    reward_extra_infos_dict.setdefault(
+                        "request_id",
+                        batch.non_tensor_batch["request_id"].tolist(),
+                    )
+
+                self._dump_generations(
+                    inputs=inputs,
+                    outputs=outputs,
+                    gts=sample_gts,
+                    scores=scores,
+                    reward_extra_infos_dict=reward_extra_infos_dict,
+                    dump_path=rollout_data_dir,
+                )
+
+    def _validate_metrics(self, is_last_step, last_val_metrics, metrics, timing_raw):
+        if (
+            self.val_reward_fn is not None
+            and self.config.trainer.test_freq > 0
+            and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0)
+        ):
+            with marked_timer("testing", timing_raw, color="green"):
+                val_metrics: dict = self._validate()
+                if is_last_step:
+                    last_val_metrics = val_metrics
+            metrics.update(val_metrics)
+            return last_val_metrics
+
+    def _collect_metrics(self, batch, epoch, metrics, timing_raw):
+        steps_duration = timing_raw["step"]
+        self.max_steps_duration = max(self.max_steps_duration, steps_duration)
+
+        # training metrics
+        metrics.update(
+            {
+                "training/global_step": self.global_steps,
+                "training/epoch": epoch,
+            }
+        )
+        # collect metrics
+        metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic))
+        metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw))
+        # TODO: implement actual tflpo and theoretical tflpo
+        n_gpus = self.resource_pool_manager.get_n_gpus()
+        metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus))
+
+    def _post_batch_processing(self, batch: DataProto):
+        # this is experimental and may be changed/removed in the future in favor of a general-purpose one
+        if isinstance(self.train_dataloader.sampler, AbstractCurriculumSampler):
+            self.train_dataloader.sampler.update(batch=batch)
+
+        # this is experimental and may be changed/removed in the future
+        # in favor of a general-purpose data buffer pool
+        if hasattr(self.train_dataset, "on_batch_end"):
+            # The dataset may be changed after each training batch
+            self.train_dataset.on_batch_end(batch=batch)

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/sglang_rollout/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/sglang_rollout/sglang_async_server.py

@@ -0,0 +1,189 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import logging
+from typing import Any, Optional
+
+import ray
+import torch
+from ray.actor import ActorHandle
+
+from verl.workers.config import HFModelConfig, RewardModelConfig, RolloutConfig
+from verl.workers.rollout.replica import RolloutMode
+from verl.workers.rollout.sglang_rollout.async_sglang_server import (
+    SGLangHttpServer,
+    SGLangReplica,
+)
+
+logger = logging.getLogger(__file__)
+logger.setLevel(logging.INFO)
+
+
+class SGLangHttpServerForPartial(SGLangHttpServer):
+    def __init__(
+        self,
+        config: RolloutConfig | RewardModelConfig,
+        model_config: HFModelConfig,
+        rollout_mode: RolloutMode,
+        workers: list[ActorHandle],
+        replica_rank: int,
+        node_rank: int,
+        nnodes: int,
+        cuda_visible_devices: str,
+        base_gpu_id: int,
+    ):
+        super().__init__(
+            config=config,
+            model_config=model_config,
+            rollout_mode=rollout_mode,
+            workers=workers,
+            replica_rank=replica_rank,
+            node_rank=node_rank,
+            nnodes=nnodes,
+            cuda_visible_devices=cuda_visible_devices,
+            base_gpu_id=base_gpu_id,
+        )
+
+        # for cancel LLMServer
+        self.paused = False
+        self.lock = asyncio.Lock()
+        self.cancel_event: dict[str, asyncio.Event] = {}
+        self.req_output: dict[str, Optional[dict[str, Any]]] = {}
+
+    async def _generate_step(
+        self,
+        prompt_ids: torch.Tensor,
+        sampling_params: dict[str, Any],
+        request_id: str,
+        image_data: Optional[list[Any]] = None,
+    ) -> None:
+        sampling_params = dict(sampling_params)
+
+        max_new_tokens = min(
+            self.config.response_length,
+            self.config.max_model_len - len(prompt_ids) - 1,
+        )
+        sampling_params["max_new_tokens"] = max_new_tokens
+
+        sampling_params.setdefault(
+            "repetition_penalty",
+            self.config.get("repetition_penalty", 1.0),
+        )
+
+        sampling_params.pop("logprobs", None)
+        return_logprob = True
+        from sglang.srt.managers.io_struct import GenerateReqInput
+
+        request = GenerateReqInput(
+            rid=request_id,
+            input_ids=prompt_ids,
+            sampling_params=sampling_params,
+            return_logprob=return_logprob,
+            image_data=image_data,
+        )
+        generator = self.tokenizer_manager.generate_request(request, None)
+        async for output in generator:
+            self.req_output[request_id] = output
+
+        assert self.req_output[request_id] is not None
+
+    async def generate_for_partial(
+        self,
+        prompt_ids: torch.Tensor,
+        sampling_params: dict[str, Any],
+        request_id: str,
+        image_data: Optional[list[Any]] = None,
+    ) -> tuple[list[int], list[float], bool]:
+        async with self.lock:
+            if self.paused:
+                return [], [], True
+            self.req_output[request_id] = None
+            self.cancel_event[request_id] = asyncio.Event()
+            cancel_handle = asyncio.create_task(self.cancel_event[request_id].wait())
+            generation_handle = asyncio.create_task(
+                self._generate_step(prompt_ids, sampling_params, request_id, image_data)
+            )
+        done, pending = await asyncio.wait(
+            [generation_handle, cancel_handle],
+            return_when=asyncio.FIRST_COMPLETED,
+        )
+        for task in done:
+            await task
+
+        for task in pending:
+            task.cancel()
+        async with self.lock:
+            output = self.req_output.get(request_id)
+            if output is None:
+                self.cancel_event.pop(request_id, None)
+                self.req_output.pop(request_id, None)
+                return [], [], True
+            meta_info = output.get("meta_info", {})
+            output_token_logprobs = meta_info.get("output_token_logprobs")
+
+            token_ids: list[int] = []
+            log_probs: list[float] = []
+
+            if output_token_logprobs is not None:
+                for log_prob, token_id, _ in output_token_logprobs:
+                    token_ids.append(int(token_id))
+                    log_probs.append(float(log_prob))
+            else:
+                token_ids = list(output["output_ids"])
+                log_probs = []
+            is_cancel = generation_handle not in done
+            self.cancel_event.pop(request_id, None)
+            self.req_output.pop(request_id, None)
+
+        return token_ids, log_probs, is_cancel
+
+    async def cancel(self):
+        async with self.lock:
+            self.paused = True
+            for request_id in self.cancel_event:
+                self.cancel_event[request_id].set()
+
+    async def resume(self):
+        async with self.lock:
+            self.paused = False
+
+    async def reset_prefix_cache(self):
+        async with self.lock:
+            print("Reset prefix cache ...")
+            await self.tokenizer_manager.flush_cache()
+
+
+class FullyAsyncSGLangReplica(SGLangReplica):
+    def __init__(
+        self,
+        replica_rank: int,
+        config: RolloutConfig | RewardModelConfig,
+        model_config: HFModelConfig,
+        gpus_per_node: int = 8,
+        is_reward_model: bool = False,
+    ):
+        super().__init__(replica_rank, config, model_config, gpus_per_node, is_reward_model)
+        self.server_class = ray.remote(SGLangHttpServerForPartial)
+
+    async def cancel(self):
+        """Cancel each rollout server."""
+        await asyncio.gather(*[server.cancel.remote() for server in self.servers])
+
+    async def resume(self):
+        """Resume each rollout server."""
+        await asyncio.gather(*[server.resume.remote() for server in self.servers])
+
+    async def reset_prefix_cache(self):
+        """reset kv cache in each rollout server."""
+        await asyncio.gather(*[server.reset_prefix_cache.remote() for server in self.servers])

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_30b_a3b_base_math_fsdp.sh

@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO-Qwen3-30B-A3B-Base-Async'
+exp_name='Fsdp2-tp4sp4'
+
+# Ray
+RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+DATA_PATH=${RAY_DATA_HOME:-"${HOME}/verl"}
+DATA_PATH=${DATA_PATH:-"/mnt/bn/${BYTENAS}"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${DATA_PATH}/shared/models/Qwen3-30B-A3B-Base"}
+CKPTS_DIR=${CKPTS_DIR:-"${DATA_PATH}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${DATA_PATH}/shared/data/dapo-math/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${DATA_PATH}/shared/data/dapo-math/aime-2024.parquet"}
+
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 20))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+enable_filter_groups=True
+filter_groups_metric=acc
+max_num_gen_batches=10
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+
+NNODES=${NNODES:-4}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+# Fully async specific parameters
+n_gpus_rollout=8
+n_gpus_training=8 
+n_nodes_rollout=2 
+n_nodes_train=2 # $((NNODES - n_nodes_rollout))
+
+train_bsz=512
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((train_bsz * 400)))
+test_freq=25
+staleness_threshold=0.6 # 0 0.3 1
+require_batches=1
+total_train_gpus=$((n_gpus_training * n_nodes_train))
+total_rollout_gpus=$((n_gpus_rollout * n_nodes_rollout))
+trigger_parameter_sync_step=$((train_bsz / ( train_prompt_mini_bsz * require_batches))) # 8 16 32
+partial_rollout=True
+enforce_eager=False
+nccl_timeout=72000
+enable_sleep_mode=False
+
+# Performance Related Parameter
+sp_size=4
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$((max_prompt_length + max_response_length))
+infer_ppo_max_token_len=$((max_prompt_length + max_response_length))
+ref_offload=True
+actor_offload=False
+gen_tp=4
+fsdp_size=-1
+
+
+ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \
+    --working-dir "${WORKING_DIR}" \
+    --address "${RAY_ADDRESS}" \
+    -- python3 -m verl.experimental.fully_async_policy.fully_async_main \
+    --config-path=config \
+    --config-name='fully_async_dapo_trainer.yaml' \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    actor_rollout_ref.actor.strategy=fsdp \
+    critic.strategy=fsdp \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    actor_rollout_ref.nccl_timeout=${nccl_timeout} \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.50 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    +actor_rollout_ref.rollout.enable_sleep_mode=${enable_sleep_mode} \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.enforce_eager=${enforce_eager} \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    reward_model.reward_manager=dapo \
+    reward_model.overlong_buffer.enable=${enable_overlong_buffer} \
+    reward_model.overlong_buffer.len=${overlong_buffer_len} \
+    reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','wandb'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}-i${total_rollout_gpus}_t${total_train_gpus}_s${staleness_threshold}" \
+    trainer.val_before_train=True \
+    trainer.test_freq="${test_freq}" \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${n_nodes_train}" \
+    trainer.n_gpus_per_node="${n_gpus_training}" \
+    rollout.nnodes="${n_nodes_rollout}" \
+    rollout.n_gpus_per_node="${n_gpus_rollout}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.test_freq=${test_freq} \
+    rollout.total_epochs=10 \
+    async_training.require_batches=${require_batches} \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_async_retool.sh

@@ -0,0 +1,141 @@
+set -x
+
+export VLLM_USE_V1=1
+
+# ================= data/model/tool =================
+HDFS_ROOT=${HDFS_ROOT:-$PWD}
+DATA_ROOT=${DATA_ROOT:-$PWD}
+
+dapo_math_17k=$DATA_ROOT/dataset/BytedTsinghua-SIA/DAPO-Math-17k
+aime_2024=$DATA_ROOT/dataset/Maxwell-Jia/AIME_2024
+aime_2025=$DATA_ROOT/dataset/yentinglin/aime_2025
+model_path=$HDFS_ROOT/checkpoint/multiturn-sft-qwen-2.5-7b-instruct/global_step_372
+
+train_files="['$dapo_math_17k']"
+test_files="['$aime_2025', '$aime_2024']"
+
+# tool
+tool_config_path=recipe/retool/sandbox_fusion_tool_config.yaml
+retool_path=recipe/retool/retool.py
+
+# wandb / tensorboard
+project_name=retool
+experiment_name=qwen2.5-7b_dapo_async_tool
+default_local_dir=$DATA_ROOT/checkpoint/$experiment_name
+
+# ================= algorithm =================
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+max_turns=16
+max_prompt_length=2048
+max_response_length=16384
+actor_lr=1e-6
+
+# ================= perfomance =================
+infer_tp=4 # vllm
+train_sp=4 # train
+fsdp_size=4 # train
+offload=False
+
+actor_max_token_len_per_gpu=$(( (max_prompt_length + max_response_length) * 1 ))
+log_prob_max_token_len_per_gpu=$(( actor_max_token_len_per_gpu * 4 ))
+
+# ================= async policy =================
+rollout_name="vllm"
+rollout_mode="async"
+
+NNODES=${NNODES:-1}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+n_gpus_rollout=4
+n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout))
+
+train_batch_size=0
+ppo_mini_batch_size=16
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+n_resp_per_prompt_val=30
+total_rollout_steps=$(((64*250)))
+test_freq=10
+staleness_threshold=0.5
+trigger_parameter_sync_step=4
+require_batches=1
+partial_rollout=True
+
+python3 -m verl.experimental.fully_async_policy.fully_async_main \
+    algorithm.adv_estimator=$adv_estimator \
+    algorithm.use_kl_in_reward=$use_kl_in_reward \
+    algorithm.kl_ctrl.kl_coef=$kl_coef \
+    data.train_files="$train_files" \
+    data.val_files="$test_files" \
+    data.return_raw_chat=True \
+    data.train_batch_size=$train_batch_size \
+    data.max_prompt_length=$max_prompt_length \
+    data.max_response_length=$max_response_length \
+    data.filter_overlong_prompts=True \
+    data.truncation='error' \
+    data.custom_cls.path=$retool_path \
+    data.custom_cls.name=CustomRLHFDataset \
+    custom_reward_function.path=$retool_path \
+    custom_reward_function.name=compute_score \
+    actor_rollout_ref.hybrid_engine=False \
+    actor_rollout_ref.model.path=$model_path \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.use_kl_loss=$use_kl_loss \
+    actor_rollout_ref.actor.kl_loss_coef=$kl_loss_coef \
+    actor_rollout_ref.actor.clip_ratio_low=$clip_ratio_low \
+    actor_rollout_ref.actor.clip_ratio_high=$clip_ratio_high \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.actor.optim.lr=$actor_lr \
+    actor_rollout_ref.actor.use_dynamic_bsz=True \
+    actor_rollout_ref.actor.ppo_mini_batch_size=$ppo_mini_batch_size \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$actor_max_token_len_per_gpu \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=$train_sp \
+    actor_rollout_ref.actor.fsdp_config.param_offload=$offload \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=$offload \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=$log_prob_max_token_len_per_gpu \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.mode=async \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=$infer_tp \
+    actor_rollout_ref.rollout.multi_turn.enable=True \
+    actor_rollout_ref.rollout.multi_turn.max_user_turns=$max_turns \
+    actor_rollout_ref.rollout.multi_turn.max_assistant_turns=$max_turns \
+    actor_rollout_ref.rollout.multi_turn.tool_config_path=$tool_config_path \
+    actor_rollout_ref.rollout.multi_turn.format=hermes \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \
+    actor_rollout_ref.rollout.n=$n_resp_per_prompt \
+    actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \
+    actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \
+    actor_rollout_ref.rollout.val_kwargs.n=$n_resp_per_prompt_val \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name=$project_name \
+    trainer.experiment_name=$experiment_name \
+    trainer.val_before_train=True \
+    trainer.log_val_generations=20 \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir=$default_local_dir \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    trainer.nnodes=$NNODES \
+    trainer.n_gpus_per_node=$n_gpus_training \
+    rollout.nnodes=$NNODES \
+    rollout.n_gpus_per_node=$n_gpus_rollout \
+    rollout.total_rollout_steps=$total_rollout_steps \
+    rollout.total_epochs=10 \
+    rollout.test_freq=$test_freq \
+    async_training.staleness_threshold=$staleness_threshold \
+    async_training.trigger_parameter_sync_step=$trigger_parameter_sync_step \
+    async_training.require_batches=$require_batches \
+    async_training.partial_rollout=$partial_rollout \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_math_fsdp2_16_16.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO'
+exp_name='dapo_qwen2-7B-math_28k_fsdp2_fully-async_16-16'
+
+# Ray
+# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+# WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 28))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3))
+ref_offload=True
+actor_offload=False
+gen_tp=4
+sp_size=4
+fsdp_size=8
+
+# Fully async specific parameters
+NNODES_ROLLOUT=${NNODES_ROLLOUT:-2}
+NNODES_TRAIN=${NNODES_TRAIN:-2}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*400)))
+test_freq=20
+staleness_threshold=0.1
+trigger_parameter_sync_step=4
+require_batches=4
+partial_rollout=True
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_math_fsdp2_32_32.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO'
+exp_name='dapo_qwen2-7B-math_28k_fsdp2_fully-async_32-32'
+
+# Ray
+# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+# WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 28))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3))
+ref_offload=True
+actor_offload=False
+gen_tp=4
+sp_size=4
+fsdp_size=8
+
+# Fully async specific parameters
+NNODES_ROLLOUT=${NNODES_ROLLOUT:-4}
+NNODES_TRAIN=${NNODES_TRAIN:-4}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*400)))
+test_freq=20
+staleness_threshold=0.1
+trigger_parameter_sync_step=4
+require_batches=4
+partial_rollout=True
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_math_fsdp2_4_12.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO'
+exp_name='DAPO-Qwen2.5-7b-MATH-0527a1-fsdp2-fully-async-4-12'
+
+# Ray
+# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+# WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 8))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3))
+ref_offload=True
+actor_offload=False
+gen_tp=1
+sp_size=1
+fsdp_size=2
+
+# Fully async specific parameters
+NNODES=${NNODES:-2}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+n_gpus_rollout=2
+n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout))
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*100)))
+test_freq=10
+staleness_threshold=0.1
+trigger_parameter_sync_step=4
+require_batches=4
+partial_rollout=True
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.test_freq="${test_freq}" \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES}" \
+    trainer.n_gpus_per_node="${n_gpus_training}" \
+    rollout.nnodes="${NNODES}" \
+    rollout.n_gpus_per_node="${n_gpus_rollout}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_math_fsdp2_4_4.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO'
+exp_name='DAPO-Qwen2.5-7b-MATH-0527a1-fsdp2-fully-async-4-4'
+
+# Ray
+# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+# WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 8))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3))
+ref_offload=True
+actor_offload=False
+gen_tp=1
+sp_size=1
+fsdp_size=2
+
+# Fully async specific parameters
+NNODES=${NNODES:-1}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+n_gpus_rollout=4
+n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout))
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*100)))
+test_freq=10
+staleness_threshold=0.1
+trigger_parameter_sync_step=4
+require_batches=4
+partial_rollout=True
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=False \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES}" \
+    trainer.n_gpus_per_node="${n_gpus_training}" \
+    rollout.nnodes="${NNODES}" \
+    rollout.n_gpus_per_node="${n_gpus_rollout}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_math_fsdp2_64_64.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO'
+exp_name='dapo_qwen2-7B-math_28k_fsdp2_fully-async_64-64'
+
+# Ray
+# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+# WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 28))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3))
+ref_offload=True
+actor_offload=False
+gen_tp=4
+sp_size=4
+fsdp_size=8
+
+# Fully async specific parameters
+NNODES_ROLLOUT=${NNODES_ROLLOUT:-8}
+NNODES_TRAIN=${NNODES_TRAIN:-8}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*400)))
+test_freq=20
+staleness_threshold=0.5
+trigger_parameter_sync_step=4
+require_batches=4
+partial_rollout=True
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_math_fsdp2_64_64_mis.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO'
+exp_name='dapo_qwen2-7B-math_28k_fsdp2_fully-async_64-64'
+
+# Ray
+# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+# WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 28))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3))
+ref_offload=True
+actor_offload=False
+gen_tp=4
+sp_size=4
+fsdp_size=8
+
+# Fully async specific parameters
+NNODES_ROLLOUT=${NNODES_ROLLOUT:-8}
+NNODES_TRAIN=${NNODES_TRAIN:-8}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*400)))
+test_freq=20
+staleness_threshold=0.5
+trigger_parameter_sync_step=4
+require_batches=4
+partial_rollout=True
+
+# Rollout Correction
+rollout_is=token
+rollout_is_threshold=2.0
+rollout_rs=seq_mean_k1
+rollout_rs_threshold="0.99_1.001"
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True \
+    async_training.compute_prox_log_prob=True \
+    algorithm.rollout_correction.rollout_is=${rollout_is} \
+    algorithm.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \
+    algorithm.rollout_correction.rollout_rs=${rollout_rs} \
+    algorithm.rollout_correction.rollout_rs_threshold=${rollout_rs_threshold}

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/dapo_7b_math_fsdp2_8_8.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='DAPO'
+exp_name='DAPO-Qwen2.5-7b-MATH-0527a1-fsdp2-fully-async-8-8'
+
+# Ray
+# RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+# WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+# RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+# Paths
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+# very important! please modify the max_position_embeddings in config.json to 32768 after downloading from huggingface
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=False
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 8))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# Training parameters
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 2))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) * 3))
+ref_offload=True
+actor_offload=False
+gen_tp=1
+sp_size=1
+fsdp_size=2
+
+# Fully async specific parameters
+NNODES_ROLLOUT=${NNODES_ROLLOUT:-1}
+NNODES_TRAIN=${NNODES_TRAIN:-1}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=32
+total_rollout_steps=$(((512*100)))
+test_freq=10
+staleness_threshold=0.1
+trigger_parameter_sync_step=4
+require_batches=4
+partial_rollout=True
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.actor.strategy=fsdp2 \
+    critic.strategy=fsdp2 \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.hybrid_engine=False \
+    +actor_rollout_ref.model.override_config.max_position_embeddings=32768 \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.fsdp_config.param_offload=${actor_offload} \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=${actor_offload} \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=1.0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=${ref_offload} \
+    actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \
+    actor_rollout_ref.actor.fsdp_config.fsdp_size=${fsdp_size} \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.save_freq=-1 \
+    trainer.default_local_dir="${CKPTS_DIR}" \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/geo3k_qwen25vl_7b_megatron_4_4.sh

@@ -0,0 +1,111 @@
+set -x
+ENGINE=${1:-vllm}
+export CUDA_DEVICE_MAX_CONNECTIONS=1 # For megatron communication/computation overlapping
+
+
+HF_MODEL_PATH=${HF_MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-VL-7B-Instruct"}
+
+train_path=$HOME/data/geo3k/train.parquet
+test_path=$HOME/data/geo3k/test.parquet
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+
+# Fully async specific parameters
+NNODES=${NNODES:-1}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+n_gpus_rollout=4
+n_gpus_training=$((NGPUS_PER_NODE - n_gpus_rollout))
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=4
+train_prompt_mini_bsz=128
+total_rollout_steps=$(((512*100)))
+test_freq=5
+staleness_threshold=0.1
+trigger_parameter_sync_step=4
+require_batches=2
+partial_rollout=True
+total_epochs=200
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    --config-path=config \
+    --config-name='fully_async_ppo_megatron_trainer.yaml'\
+    algorithm.adv_estimator=grpo \
+    data.train_files="$train_path" \
+    data.val_files="$test_path" \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.max_prompt_length=1024 \
+    data.max_response_length=2048 \
+    actor_rollout_ref.rollout.max_model_len=32768 \
+    actor_rollout_ref.rollout.max_num_batched_tokens=32768 \
+    data.filter_overlong_prompts=True \
+    data.truncation='error' \
+    data.gen_batch_size=${gen_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.model.path=$HF_MODEL_PATH \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_decay_steps=51200 \
+    actor_rollout_ref.hybrid_engine=False \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \
+    actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=1 \
+    actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \
+    actor_rollout_ref.actor.use_kl_loss=True \
+    actor_rollout_ref.actor.kl_loss_coef=0.01 \
+    actor_rollout_ref.actor.kl_loss_type=low_var_kl \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
+    actor_rollout_ref.actor.use_dynamic_bsz=True \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=5120 \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=5120 \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=5120 \
+    actor_rollout_ref.rollout.name=$ENGINE \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    +actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \
+    actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=1 \
+    actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=1 \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \
+    actor_rollout_ref.actor.megatron.use_mbridge=True \
+    actor_rollout_ref.actor.megatron.param_offload=True \
+    actor_rollout_ref.actor.megatron.optimizer_offload=True \
+    actor_rollout_ref.actor.megatron.grad_offload=True \
+    actor_rollout_ref.ref.megatron.param_offload=True \
+    algorithm.use_kl_in_reward=False \
+    trainer.critic_warmup=0 \
+    trainer.logger='["console","wandb"]' \
+    trainer.project_name='verl_grpo_example_geo3k' \
+    trainer.experiment_name='qwen2_5_vl_7b_megatron_async' \
+    trainer.test_freq="${test_freq}" \
+    trainer.total_epochs="${total_epochs}" \
+    trainer.val_before_train=False \
+    trainer.save_freq=-1 \
+    trainer.resume_mode=auto \
+    trainer.nnodes="${NNODES}" \
+    trainer.n_gpus_per_node="${n_gpus_training}" \
+    rollout.nnodes="${NNODES}" \
+    rollout.n_gpus_per_node="${n_gpus_rollout}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs="${total_epochs}" \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/grpo_30b_a3b_base_math_megatron_96_32.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='GRPO-Qwen3-30b-Base-MATH'
+exp_name='GRPO-Qwen3-30b-Base-MATH-megatron-fully-async_96-32'
+
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-Base"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=True
+kl_loss_coef=0.001
+kl_loss_type=low_var_kl
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 8))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length)))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length)))
+offload=True
+train_ppo_micro_batch_size_per_gpu=2
+infer_ppo_micro_batch_size_per_gpu=2
+
+optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.}
+
+COMMON_PP=${COMMON_PP:-1}
+COMMON_VPP=${COMMON_VPP:-null}
+COMMON_CP=${COMMON_CP:-2}
+COMMON_TP=${COMMON_TP:-2}
+COMMON_EP=${COMMON_EP:-8}
+COMMON_ETP=${COMMON_ETP:-1}
+
+TRAIN_TP=${TRAIN_TP:-$COMMON_TP}
+INFER_TP=${INFER_TP:-4}
+
+ACTOR_PP=${ACTOR_PP:-$COMMON_PP}
+ACTOR_VPP=${ACTOR_VPP:-$COMMON_VPP}
+ACTOR_CP=${ACTOR_CP:-$COMMON_CP}
+ACTOR_TP=${ACTOR_TP:-$TRAIN_TP}
+ACTOR_EP=${ACTOR_EP:-$COMMON_EP}
+ACTOR_ETP=${ACTOR_ETP:-$COMMON_ETP}
+ROLLOUT_TP=${ROLLOUT_TP:-$INFER_TP}
+REF_PP=${REF_PP:-$COMMON_PP}
+REF_VPP=${REF_VPP:-$COMMON_VPP}
+REF_CP=${REF_CP:-$COMMON_CP}
+REF_TP=${REF_TP:-$TRAIN_TP}
+REF_EP=${REF_EP:-$COMMON_EP}
+REF_ETP=${REF_ETP:-$COMMON_ETP}
+CRITIC_PP=${CRITIC_PP:-$COMMON_PP}
+CRITIC_VPP=${CRITIC_VPP:-$COMMON_VPP}
+CRITIC_CP=${CRITIC_CP:-$COMMON_CP}
+CRITIC_TP=${CRITIC_TP:-$TRAIN_TP}
+CRITIC_EP=${CRITIC_EP:-$COMMON_EP}
+CRITIC_ETP=${CRITIC_ETP:-$COMMON_ETP}
+RM_PP=${RM_PP:-$COMMON_PP}
+RM_VPP=${RM_VPP:-$COMMON_VPP}
+RM_CP=${RM_CP:-$COMMON_CP}
+RM_TP=${RM_TP:-$TRAIN_TP}
+RM_EP=${RM_EP:-$COMMON_EP}
+RM_ETP=${RM_ETP:-$COMMON_ETP}
+
+# install mbridge
+# pip3 install git+https://github.com/ISEEKYAN/mbridge
+USE_MBRIDGE=True
+USE_DIST_CKPT=False
+
+# Fully async specific parameters
+NNODES_ROLLOUT=${NNODES_ROLLOUT:-12}
+NNODES_TRAIN=${NNODES_TRAIN:-4}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=128
+total_rollout_steps=$(((512*400)))
+test_freq=20
+staleness_threshold=0.5
+trigger_parameter_sync_step=4
+require_batches=1
+partial_rollout=True
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    --config-path=config \
+    --config-name='fully_async_ppo_megatron_trainer.yaml'\
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.model.use_fused_kernels=False \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_ppo_micro_batch_size_per_gpu} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.lr_decay_style='constant' \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.optim.lr_decay_steps=${total_rollout_steps} \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \
+    actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \
+    actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \
+    actor_rollout_ref.actor.megatron.param_offload=${offload} \
+    actor_rollout_ref.actor.megatron.grad_offload=${offload} \
+    actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \
+    actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${ACTOR_TP} \
+    actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${ACTOR_PP} \
+    actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=${ACTOR_VPP} \
+    actor_rollout_ref.actor.megatron.context_parallel_size=${ACTOR_CP} \
+    actor_rollout_ref.actor.megatron.expert_model_parallel_size=${ACTOR_EP} \
+    actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=${ACTOR_ETP} \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type="flex" \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_enable_deepep=True \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${INFER_TP} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    actor_rollout_ref.hybrid_engine=False \
+    actor_rollout_ref.rollout.enforce_eager=True \
+    actor_rollout_ref.rollout.free_cache_engine=True \
+    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \
+    actor_rollout_ref.ref.megatron.param_offload=${offload} \
+    actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${REF_TP} \
+    actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${REF_PP} \
+    actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=${REF_VPP} \
+    actor_rollout_ref.ref.megatron.context_parallel_size=${REF_CP} \
+    actor_rollout_ref.ref.megatron.expert_model_parallel_size=${REF_EP} \
+    actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=${REF_ETP} \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.save_freq=-1 \
+    trainer.total_epochs=10 \
+    trainer.resume_mode=auto \
+    trainer.log_val_generations=10 \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True \
+

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/grpo_30b_a3b_base_math_megatron_96_32_mis.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+project_name='GRPO-Qwen3-30b-Base-MATH'
+exp_name='GRPO-Qwen3-30b-Base-MATH-megatron-fully-async_96-32'
+
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-30B-A3B-Base"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"}
+
+rollout_mode="async"
+rollout_name="vllm" # sglang or vllm
+if [ "$rollout_mode" = "async" ]; then
+    export VLLM_USE_V1=1
+    return_raw_chat="True"
+fi
+# Algorithm parameters
+adv_estimator=grpo
+
+use_kl_in_reward=False
+kl_coef=0.0
+use_kl_loss=True
+kl_loss_coef=0.001
+kl_loss_type=low_var_kl
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+# Response length parameters
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 8))
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+loss_agg_mode="token-mean"
+
+# Algorithm
+temperature=1.0
+top_p=1.0
+top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+val_top_p=0.7
+
+# Performance Related Parameter
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$(((max_prompt_length + max_response_length)))
+infer_ppo_max_token_len=$(((max_prompt_length + max_response_length)))
+offload=True
+train_ppo_micro_batch_size_per_gpu=2
+infer_ppo_micro_batch_size_per_gpu=2
+
+optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.}
+
+COMMON_PP=${COMMON_PP:-1}
+COMMON_VPP=${COMMON_VPP:-null}
+COMMON_CP=${COMMON_CP:-2}
+COMMON_TP=${COMMON_TP:-2}
+COMMON_EP=${COMMON_EP:-8}
+COMMON_ETP=${COMMON_ETP:-1}
+
+TRAIN_TP=${TRAIN_TP:-$COMMON_TP}
+INFER_TP=${INFER_TP:-4}
+
+ACTOR_PP=${ACTOR_PP:-$COMMON_PP}
+ACTOR_VPP=${ACTOR_VPP:-$COMMON_VPP}
+ACTOR_CP=${ACTOR_CP:-$COMMON_CP}
+ACTOR_TP=${ACTOR_TP:-$TRAIN_TP}
+ACTOR_EP=${ACTOR_EP:-$COMMON_EP}
+ACTOR_ETP=${ACTOR_ETP:-$COMMON_ETP}
+ROLLOUT_TP=${ROLLOUT_TP:-$INFER_TP}
+REF_PP=${REF_PP:-$COMMON_PP}
+REF_VPP=${REF_VPP:-$COMMON_VPP}
+REF_CP=${REF_CP:-$COMMON_CP}
+REF_TP=${REF_TP:-$TRAIN_TP}
+REF_EP=${REF_EP:-$COMMON_EP}
+REF_ETP=${REF_ETP:-$COMMON_ETP}
+CRITIC_PP=${CRITIC_PP:-$COMMON_PP}
+CRITIC_VPP=${CRITIC_VPP:-$COMMON_VPP}
+CRITIC_CP=${CRITIC_CP:-$COMMON_CP}
+CRITIC_TP=${CRITIC_TP:-$TRAIN_TP}
+CRITIC_EP=${CRITIC_EP:-$COMMON_EP}
+CRITIC_ETP=${CRITIC_ETP:-$COMMON_ETP}
+RM_PP=${RM_PP:-$COMMON_PP}
+RM_VPP=${RM_VPP:-$COMMON_VPP}
+RM_CP=${RM_CP:-$COMMON_CP}
+RM_TP=${RM_TP:-$TRAIN_TP}
+RM_EP=${RM_EP:-$COMMON_EP}
+RM_ETP=${RM_ETP:-$COMMON_ETP}
+
+# install mbridge
+# pip3 install git+https://github.com/ISEEKYAN/mbridge
+USE_MBRIDGE=True
+USE_DIST_CKPT=False
+
+# Fully async specific parameters
+NNODES_ROLLOUT=${NNODES_ROLLOUT:-12}
+NNODES_TRAIN=${NNODES_TRAIN:-4}
+NGPUS_PER_NODE=${NGPUS_PER_NODE:-8}
+
+train_prompt_bsz=0
+gen_prompt_bsz=1
+n_resp_per_prompt=16
+train_prompt_mini_bsz=128
+total_rollout_steps=$(((512*400)))
+test_freq=20
+staleness_threshold=0.5
+trigger_parameter_sync_step=4
+require_batches=1
+partial_rollout=True
+
+# Rollout Importance Sampling
+
+rollout_is=null
+rollout_rs=seq_mean_k1
+rollout_rs_threshold="0.999_1.001"
+
+python -m verl.experimental.fully_async_policy.fully_async_main \
+    --config-path=config \
+    --config-name='fully_async_ppo_megatron_trainer.yaml'\
+    data.train_files="${TRAIN_FILE}" \
+    data.val_files="${TEST_FILE}" \
+    data.prompt_key=prompt \
+    data.truncation='left' \
+    data.max_prompt_length=${max_prompt_length} \
+    data.max_response_length=${max_response_length} \
+    data.train_batch_size=${train_prompt_bsz} \
+    data.return_raw_chat=${return_raw_chat} \
+    actor_rollout_ref.rollout.n=${n_resp_per_prompt} \
+    algorithm.adv_estimator=${adv_estimator} \
+    algorithm.use_kl_in_reward=${use_kl_in_reward} \
+    algorithm.kl_ctrl.kl_coef=${kl_coef} \
+    async_training.compute_prox_log_prob=True \
+    algorithm.rollout_correction.rollout_is=${rollout_is} \
+    algorithm.rollout_correction.rollout_rs=${rollout_rs} \
+    algorithm.rollout_correction.rollout_rs_threshold=${rollout_rs_threshold} \
+    actor_rollout_ref.model.path="${MODEL_PATH}" \
+    actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \
+    actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.clip_ratio_c=10.0 \
+    +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.model.use_fused_kernels=False \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \
+    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${train_ppo_micro_batch_size_per_gpu} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
+    actor_rollout_ref.actor.optim.lr_decay_style='constant' \
+    actor_rollout_ref.actor.optim.weight_decay=0.1 \
+    actor_rollout_ref.actor.optim.lr_decay_steps=${total_rollout_steps} \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \
+    +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \
+    actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \
+    actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \
+    actor_rollout_ref.actor.megatron.param_offload=${offload} \
+    actor_rollout_ref.actor.megatron.grad_offload=${offload} \
+    actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \
+    actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${ACTOR_TP} \
+    actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${ACTOR_PP} \
+    actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=${ACTOR_VPP} \
+    actor_rollout_ref.actor.megatron.context_parallel_size=${ACTOR_CP} \
+    actor_rollout_ref.actor.megatron.expert_model_parallel_size=${ACTOR_EP} \
+    actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=${ACTOR_ETP} \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type="flex" \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \
+    +actor_rollout_ref.actor.megatron.override_transformer_config.moe_enable_deepep=True \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \
+    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=${INFER_TP} \
+    actor_rollout_ref.rollout.enable_chunked_prefill=True \
+    actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \
+    actor_rollout_ref.rollout.temperature=${temperature} \
+    actor_rollout_ref.rollout.top_p=${top_p} \
+    actor_rollout_ref.rollout.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \
+    actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \
+    actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.val_kwargs.n=1 \
+    actor_rollout_ref.rollout.name=${rollout_name} \
+    actor_rollout_ref.rollout.mode=${rollout_mode} \
+    actor_rollout_ref.rollout.calculate_log_probs=True \
+    actor_rollout_ref.hybrid_engine=False \
+    actor_rollout_ref.rollout.enforce_eager=True \
+    actor_rollout_ref.rollout.free_cache_engine=True \
+    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${infer_ppo_micro_batch_size_per_gpu} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \
+    actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \
+    actor_rollout_ref.ref.megatron.param_offload=${offload} \
+    actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${REF_TP} \
+    actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${REF_PP} \
+    actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=${REF_VPP} \
+    actor_rollout_ref.ref.megatron.context_parallel_size=${REF_CP} \
+    actor_rollout_ref.ref.megatron.expert_model_parallel_size=${REF_EP} \
+    actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=${REF_ETP} \
+    reward_model.reward_manager=dapo \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \
+    +reward_model.reward_kwargs.overlong_buffer_cfg.log=False \
+    +reward_model.reward_kwargs.max_resp_len=${max_response_length} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.project_name="${project_name}" \
+    trainer.experiment_name="${exp_name}" \
+    trainer.val_before_train=True \
+    trainer.save_freq=-1 \
+    trainer.total_epochs=10 \
+    trainer.resume_mode=auto \
+    trainer.log_val_generations=10 \
+    trainer.nnodes="${NNODES_TRAIN}" \
+    trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.nnodes="${NNODES_ROLLOUT}" \
+    rollout.n_gpus_per_node="${NGPUS_PER_NODE}" \
+    rollout.total_rollout_steps="${total_rollout_steps}" \
+    rollout.total_epochs=10 \
+    rollout.test_freq="${test_freq}" \
+    async_training.staleness_threshold="${staleness_threshold}" \
+    async_training.trigger_parameter_sync_step="${trigger_parameter_sync_step}" \
+    async_training.require_batches="${require_batches}" \
+    async_training.partial_rollout="${partial_rollout}" \
+    async_training.use_rollout_log_probs=True \

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/shell/runtime_env.yaml

@@ -0,0 +1,4 @@
+env_vars:
+  VLLM_USE_V1: "1"
+  NCCL_DEBUG: "INFO"
+  HYDRA_FULL_ERROR: "1"

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/unittest/simple_streaming_demo.py

@@ -0,0 +1,176 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import random
+import time
+
+
+class SimpleStreamingSystem:
+    """Simplified streaming system demonstration"""
+
+    def __init__(self, max_concurrent_tasks: int = 4):
+        self.max_concurrent_tasks = max_concurrent_tasks
+        self.data_queue = asyncio.Queue()
+        self.result_queue = asyncio.Queue()
+        self.consumer_count = 0
+
+    # Data stream coroutine
+    async def data_stream(self):
+        # Add initial data
+        # Prepare test data
+        test_data = [{"id": f"task_{i}", "content": f"data_{i}"} for i in range(8)]
+        await self.add_data_stream(test_data)
+
+        # Simulate subsequent data stream
+        await asyncio.sleep(3)
+        print("\nAdding second batch of data...")
+        extra_data = [{"id": f"extra_{i}", "content": f"extra_data_{i}"} for i in range(5)]
+        await self.add_data_stream(extra_data)
+
+        # Send termination signal
+        await asyncio.sleep(1)
+        await self.data_queue.put("DONE")
+        print("Sending termination signal")
+
+    async def add_data_stream(self, data_list: list[dict]):
+        """Simulate data stream"""
+        print("Starting to add data stream...")
+
+        for i, data_item in enumerate(data_list):
+            await self.data_queue.put(data_item)
+            print(f"Data {data_item['id']} added to pending queue")
+
+            # Simulate interval between data streams
+            if i < len(data_list) - 1:  # Don't wait after the last item
+                await asyncio.sleep(0.8)
+
+        print("Initial data stream added successfully")
+
+    async def _process_data_async(self, data_item: dict):
+        """Asynchronously process a single data item"""
+        data_id = data_item["id"]
+        content = data_item["content"]
+
+        # Simulate different processing times (1-3 seconds)
+        processing_time = random.uniform(1, 3)
+
+        print(f"    Starting to process {data_id}, estimated time {processing_time:.1f}s")
+
+        # Asynchronously wait for processing completion
+        await asyncio.sleep(processing_time)
+
+        result = {
+            "id": data_id,
+            "processed_content": f"Processed {content}",
+            "processing_time": round(processing_time, 2),
+            "completed_at": time.time(),
+        }
+
+        # Immediately put into result queue
+        await self.result_queue.put(result)
+        print(f"    {data_id} processing completed! (took {processing_time:.1f}s) -> Added to result queue")
+
+    async def _submit_worker(self):
+        """Stream submission worker coroutine"""
+        active_tasks = set()
+
+        print("Stream submitter started...")
+
+        while True:
+            # Get data to process
+            data_item = await self.data_queue.get()
+
+            if data_item == "DONE":
+                print("Received termination signal, waiting for remaining tasks to complete...")
+                if active_tasks:
+                    await asyncio.gather(*active_tasks, return_exceptions=True)
+                break
+
+            # Check concurrent limit
+            while len(active_tasks) >= self.max_concurrent_tasks:
+                print(f"Reached maximum concurrency {self.max_concurrent_tasks}, waiting for tasks to complete...")
+                done_tasks, active_tasks = await asyncio.wait(active_tasks, return_when=asyncio.FIRST_COMPLETED)
+
+                # Clean up completed tasks
+                for task in done_tasks:
+                    try:
+                        await task
+                        print(f"Task completed {task}")
+                    except Exception as e:
+                        print(f"Task execution failed: {e}")
+
+            # Immediately submit new task
+            task = asyncio.create_task(self._process_data_async(data_item), name=f"active {data_item}")
+            active_tasks.add(task)
+
+            print(f"Submitted task {data_item['id']}, current concurrency: {len(active_tasks)}")
+
+    async def _consumer_worker(self):
+        """Result consumer coroutine"""
+        print("Consumer started...")
+
+        while True:
+            try:
+                # Get processing result from result queue
+                result = await asyncio.wait_for(self.result_queue.get(), timeout=2.0)
+
+                self.consumer_count += 1
+
+                print(
+                    f"Consumed #{self.consumer_count}: {result['id']} "
+                    f"(processing time {result['processing_time']}s) - {result['processed_content']}"
+                )
+
+            except asyncio.TimeoutError:
+                print("    Consumer waiting...")
+                await asyncio.sleep(0.5)
+
+    async def run_demo(self):
+        """Run demonstration"""
+        print("=" * 60)
+        print(f"Maximum concurrency: {self.max_concurrent_tasks}")
+        print("=" * 60)
+
+        # Start core coroutines
+        stream_task = asyncio.create_task(self.data_stream())
+        submit_task = asyncio.create_task(self._submit_worker())
+        consumer_task = asyncio.create_task(self._consumer_worker())
+
+        try:
+            # Wait for data stream to complete
+            await stream_task
+            print("Data stream completed")
+
+            # Wait for processing to complete
+            await submit_task
+            print("All tasks processed")
+
+        finally:
+            # Cleanup
+            submit_task.cancel()
+            consumer_task.cancel()
+            await asyncio.gather(submit_task, consumer_task, return_exceptions=True)
+
+        print(f"\nFinal statistics: Consumed {self.consumer_count} results")
+
+
+async def main():
+    """Main function"""
+    system = SimpleStreamingSystem(max_concurrent_tasks=3)
+    await system.run_demo()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/vllm_rollout/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

code/RL_model/verl/verl_train/verl/experimental/fully_async_policy/vllm_rollout/vllm_async_server.py

@@ -0,0 +1,148 @@
+# Copyright 2025 Meituan Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import logging
+from typing import Any, Optional, Sequence
+
+import ray
+from ray.actor import ActorHandle
+from vllm import SamplingParams
+from vllm.inputs import TokensPrompt
+from vllm.outputs import RequestOutput
+
+from verl.workers.config import HFModelConfig, RolloutConfig
+from verl.workers.rollout.replica import RolloutMode
+from verl.workers.rollout.vllm_rollout.vllm_async_server import (
+    _qwen2_5_vl_dedup_image_tokens,
+    vLLMHttpServer,
+    vLLMReplica,
+)
+
+logger = logging.getLogger(__file__)
+logger.setLevel(logging.INFO)
+
+
+class vLLMHttpServerForPartial(vLLMHttpServer):
+    def __init__(
+        self,
+        config: RolloutConfig,
+        model_config: HFModelConfig,
+        rollout_mode: RolloutMode,
+        workers: list[ActorHandle],
+        replica_rank: int,
+        node_rank: int,
+        gpus_per_node: int,
+        nnodes: int,
+    ):
+        super().__init__(config, model_config, rollout_mode, workers, replica_rank, node_rank, gpus_per_node, nnodes)
+
+        # for cancel LLMServer
+        self.paused = False
+        self.lock = asyncio.Lock()
+        self.cancel_event: dict[str, asyncio.Event] = {}
+        self.req_output: dict[str, Optional[RequestOutput]] = {}
+
+    async def _generate_step(
+        self,
+        prompt_ids: list[int],
+        sampling_params: dict[str, Any],
+        request_id: str,
+        image_data: Optional[list[Any]] = None,
+    ):
+        max_tokens = self.config.max_model_len - len(prompt_ids)
+        sampling_params["logprobs"] = 1
+        sampling_params.setdefault("repetition_penalty", self.config.get("repetition_penalty", 1.0))
+        sampling_params = SamplingParams(max_tokens=max_tokens, **sampling_params)
+        prompt_ids = _qwen2_5_vl_dedup_image_tokens(prompt_ids, self.model_config.processor)
+        prompt = TokensPrompt(
+            prompt_token_ids=prompt_ids, multi_modal_data={"image": image_data} if image_data else None
+        )
+        generator = self.engine.generate(prompt=prompt, sampling_params=sampling_params, request_id=request_id)
+
+        # Get final response
+        async for output in generator:
+            self.req_output[request_id] = output
+        assert self.req_output[request_id] is not None
+
+    async def generate_for_partial(
+        self,
+        prompt_ids: list[int],
+        sampling_params: dict[str, Any],
+        request_id: str,
+        image_data: Optional[list[Any]] = None,
+    ) -> tuple[list[Any], list[Any], bool] | tuple[Sequence[int], list[float], Any]:
+        async with self.lock:
+            if self.paused:
+                # After cancel, all tasks will return directly and wait for the next submission
+                return [], [], True
+            self.req_output[request_id]: Optional[RequestOutput] = None
+            self.cancel_event[request_id] = asyncio.Event()
+            cancel_handle = asyncio.create_task(self.cancel_event[request_id].wait())
+            generation_handle = asyncio.create_task(
+                self._generate_step(prompt_ids, sampling_params, request_id, image_data)
+            )
+
+        done, pend = await asyncio.wait([generation_handle, cancel_handle], return_when=asyncio.FIRST_COMPLETED)
+
+        for task in done:
+            await task
+
+        for task in pend:
+            task.cancel()
+
+        async with self.lock:
+            if self.req_output[request_id] is None:
+                return [], [], True
+            token_ids = self.req_output[request_id].outputs[0].token_ids
+            log_probs: list[float] = []
+            for i, x in enumerate(self.req_output[request_id].outputs[0].logprobs):
+                # In sampling_params, logprobs is set to 1, which should return 1,
+                # but in practice there are multiple. Take the log_prob corresponding to token_id
+                token_id = self.req_output[request_id].outputs[0].token_ids[i]
+                log_probs.append(x[token_id].logprob)
+            is_cancel = generation_handle not in done
+            self.cancel_event.pop(request_id, None)
+            self.req_output.pop(request_id, None)
+        return token_ids, log_probs, is_cancel
+
+    async def cancel(self):
+        async with self.lock:
+            self.paused = True
+            for request_id in self.cancel_event:
+                self.cancel_event[request_id].set()
+
+    async def resume(self):
+        async with self.lock:
+            self.paused = False
+
+
+class FullyAsyncvLLMReplica(vLLMReplica):
+    def __init__(
+        self,
+        replica_rank: int,
+        config: RolloutConfig,
+        model_config: HFModelConfig,
+        gpus_per_node: int = 8,
+        is_reward_model: bool = False,
+    ):
+        super().__init__(replica_rank, config, model_config, gpus_per_node, is_reward_model)
+        self.server_class = ray.remote(vLLMHttpServerForPartial)
+
+    async def cancel(self):
+        """Cancel each rollout server."""
+        await asyncio.gather(*[server.cancel.remote() for server in self.servers])
+
+    async def resume(self):
+        """Resume each rollout server."""
+        await asyncio.gather(*[server.resume.remote() for server in self.servers])