diff --git a/scripts/train_justrl_qwen.sh b/scripts/train_justrl_qwen.sh index 0bf8cc0..e1ca392 100755 --- a/scripts/train_justrl_qwen.sh +++ b/scripts/train_justrl_qwen.sh @@ -211,6 +211,7 @@ set +e trainer.default_local_dir="${CHECKPOINT_DIR}" \ trainer.is_plot="${IS_PLOT}" \ "${VLLM_ENGINE_OVERRIDES[@]}" \ + "$@" \ 2>&1 | tee -a "${TRAIN_LOG}" STATUS=${PIPESTATUS[0]} set -e diff --git a/verl/verl/trainer/main_ppo.py b/verl/verl/trainer/main_ppo.py index 0962321..6f76e02 100644 --- a/verl/verl/trainer/main_ppo.py +++ b/verl/verl/trainer/main_ppo.py @@ -31,6 +31,30 @@ from verl.utils.config import validate_config from verl.utils.device import is_cuda_available from verl.utils.import_utils import load_extern_type +# [phase4-seed] Pilot-only seeding shim (Direct-OPD SFT-vs-RL policy-shift pilot). +# The upstream PPO path has no global RNG seeding at all (no trainer.seed / algorithm.seed), +# so the driver process and every Ray worker process start from unseeded python/numpy/torch +# RNGs. This helper is called on the driver (run_ppo) and inside the Ray TaskRunner actor; +# ActorRolloutRefWorker.init_model calls the equivalent block (separate process, separate +# import of this module is not guaranteed to have run the driver's call). +PHASE4_SEED = 42 + + +def _phase4_seed_everything(seed: int = PHASE4_SEED) -> None: + """Seed python/numpy/torch (+CUDA) RNGs in the current process.""" + import random + + import numpy as np + import torch + + os.environ.setdefault("PYTHONHASHSEED", str(seed)) + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + print(f"[phase4-seed] seeded python/numpy/torch RNGs with {seed} in pid={os.getpid()}") + @hydra.main(config_path="config", config_name="ppo_trainer", version_base=None) def main(config): @@ -52,6 +76,8 @@ def run_ppo(config, task_runner_class=None) -> None: model paths, and training hyperparameters. task_runner_class: For recipe to change TaskRunner. """ + _phase4_seed_everything() # [phase4-seed] driver process + # Check if Ray is not initialized if not ray.is_initialized(): # Initialize Ray with a local cluster configuration @@ -242,6 +268,8 @@ class TaskRunner: config: Training configuration object containing all parameters needed for setting up and running the PPO training process. """ + _phase4_seed_everything() # [phase4-seed] TaskRunner actor process + # Print the initial configuration. `resolve=True` will evaluate symbolic values. from pprint import pprint diff --git a/verl/verl/workers/fsdp_workers.py b/verl/verl/workers/fsdp_workers.py index de7893b..62e18d3 100644 --- a/verl/verl/workers/fsdp_workers.py +++ b/verl/verl/workers/fsdp_workers.py @@ -638,7 +638,9 @@ class ActorRolloutRefWorker(Worker, DistProfilerExtension): # 3. init trainer and rollout random states self.torch_random_states = get_torch_device().get_rng_state() gen_dp_rank = rollout_device_mesh["dp"].get_local_rank() - get_torch_device().manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states + # [phase4-seed] was `gen_dp_rank + 1000` (hardcoded upstream); pinned to the pilot seed + # base 42 so the rollout sampling generator is a documented function of the seed. + get_torch_device().manual_seed(gen_dp_rank + 42) # make sure all tp ranks have the same random states self.gen_random_states = get_torch_device().get_rng_state() get_torch_device().set_rng_state(self.torch_random_states) @@ -778,6 +780,16 @@ class ActorRolloutRefWorker(Worker, DistProfilerExtension): @register(dispatch_mode=Dispatch.ONE_TO_ALL) def init_model(self): + # [phase4-seed] Ray workers are separate processes: seed python/numpy/torch here too. + import random as _phase4_random + + _phase4_random.seed(42) + np.random.seed(42) + torch.manual_seed(42) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(42) + print(f"[phase4-seed] worker rank={os.environ.get('RANK', '?')} seeded python/numpy/torch with 42") + from verl.workers.actor import DataParallelPPOActor # This is used to import external_lib into the huggingface systems