Add files using upload-large-folder tool

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-18-07/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: true
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-18-07/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=True
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=True,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-18-07
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-18-07/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=True
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-20-59/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: true
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-20-59/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=True
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=True,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-20-59
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-20-59/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=True
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-24-15/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: true
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-24-15/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=True
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=True,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-24-15
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-24-15/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=True
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-26-44/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: false
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-26-44/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=False
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=False,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-26-44
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-26-44/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=False
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-33-01/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: false
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-33-01/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=False
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=False,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-33-01
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-33-01/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=False
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-35-38/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: false
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-35-38/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=False
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=False,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-35-38
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-35-38/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=False
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-41-08/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: false
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-41-08/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=False
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=False,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-41-08
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-41-08/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=False
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-42-57/.hydra/config.yaml

@@ -0,0 +1,169 @@
+data:
+  tokenizer: null
+  train_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+  val_files: /home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+  train_data_num: null
+  val_data_num: null
+  prompt_key: prompt
+  max_prompt_length: 4096
+  max_response_length: 1024
+  max_start_length: 256
+  max_obs_length: 512
+  train_batch_size: 128
+  val_batch_size: 64
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle_train_dataloader: true
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B-Instruct-2507
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: false
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 64
+    ppo_micro_batch_size: 64
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    state_masking: false
+    clip_ratio: 0.2
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: -1
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      grad_offload: false
+      optimizer_offload: true
+      fsdp_size: -1
+    ppo_micro_batch_size_per_gpu: 16
+  ref:
+    fsdp_config:
+      param_offload: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 0.95
+    prompt_length: ${data.max_prompt_length}
+    response_length: ${data.max_response_length}
+    dtype: bfloat16
+    gpu_memory_utilization: 0.4
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 8192
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: 64
+    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
+    do_sample: true
+    'n': 1
+    n_agent: 1
+critic:
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: -1
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: ${actor_rollout_ref.model.path}
+    override_config: {}
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    enable_gradient_checkpointing: false
+    use_remove_padding: false
+    fsdp_config:
+      param_offload: false
+      grad_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
+  ppo_micro_batch_size: 64
+  forward_micro_batch_size: ${critic.ppo_micro_batch_size}
+  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
+  shuffle: ${actor_rollout_ref.actor.shuffle}
+  grad_clip: 1.0
+  cliprange_value: 0.5
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: ${actor_rollout_ref.model.path}
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: ${actor_rollout_ref.model.external_lib}
+    use_remove_padding: false
+    fsdp_config:
+      min_num_params: 0
+      param_offload: false
+  micro_batch_size: 64
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: ${critic.use_dynamic_bsz}
+  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
+  structure_format_score: 0
+  final_format_score: 0
+  retrieval_score: 0
+retriever:
+  url: http://127.0.0.1:8000/retrieve
+  topk: 3
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  no_think_rl: false
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+  state_masking:
+    start_state_marker: <information>
+    end_state_marker: </information>
+trainer:
+  total_epochs: 15
+  total_training_steps: 1005
+  project_name: ''
+  experiment_name: llm_guard_3B_10k_v2
+  logger:
+  - wandb
+  nnodes: 1
+  n_gpus_per_node: 2
+  save_freq: 100
+  test_freq: 50
+  critic_warmup: 0
+  default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name}
+  default_local_dir: verl_checkpoints/llm_guard_3B_10k_v2
+max_turns: 1
+do_search: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-42-57/.hydra/hydra.yaml

@@ -0,0 +1,189 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+    - data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+    - data.train_batch_size=128
+    - data.val_batch_size=64
+    - data.max_prompt_length=4096
+    - data.max_response_length=1024
+    - data.shuffle_train_dataloader=True
+    - algorithm.adv_estimator=grpo
+    - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+    - actor_rollout_ref.model.enable_gradient_checkpointing=true
+    - actor_rollout_ref.model.use_remove_padding=False
+    - actor_rollout_ref.actor.optim.lr=1e-6
+    - actor_rollout_ref.actor.ppo_mini_batch_size=64
+    - +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+    - actor_rollout_ref.actor.fsdp_config.param_offload=true
+    - actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+    - actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+    - actor_rollout_ref.rollout.tensor_model_parallel_size=1
+    - actor_rollout_ref.rollout.name=vllm
+    - actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+    - actor_rollout_ref.ref.log_prob_micro_batch_size=64
+    - actor_rollout_ref.ref.fsdp_config.param_offload=True
+    - actor_rollout_ref.actor.kl_loss_coef=0.001
+    - trainer.logger=[wandb]
+    - trainer.n_gpus_per_node=2
+    - trainer.nnodes=1
+    - trainer.save_freq=100
+    - trainer.test_freq=50
+    - trainer.project_name=
+    - trainer.experiment_name=llm_guard_3B_10k_v2
+    - trainer.total_epochs=15
+    - trainer.total_training_steps=1005
+    - trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+    - do_search=false
+    - max_turns=1
+  job:
+    name: main_ppo
+    chdir: null
+    override_dirname: +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16,actor_rollout_ref.actor.fsdp_config.optimizer_offload=true,actor_rollout_ref.actor.fsdp_config.param_offload=true,actor_rollout_ref.actor.kl_loss_coef=0.001,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.ppo_mini_batch_size=64,actor_rollout_ref.model.enable_gradient_checkpointing=true,actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507,actor_rollout_ref.model.use_remove_padding=False,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.gpu_memory_utilization=0.4,actor_rollout_ref.rollout.log_prob_micro_batch_size=64,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.tensor_model_parallel_size=1,algorithm.adv_estimator=grpo,data.max_prompt_length=4096,data.max_response_length=1024,data.shuffle_train_dataloader=True,data.train_batch_size=128,data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet,data.val_batch_size=64,data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet,do_search=false,max_turns=1,trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2,trainer.experiment_name=llm_guard_3B_10k_v2,trainer.logger=[wandb],trainer.n_gpus_per_node=2,trainer.nnodes=1,trainer.project_name=,trainer.save_freq=100,trainer.test_freq=50,trainer.total_epochs=15,trainer.total_training_steps=1005
+    id: ???
+    num: ???
+    config_name: ppo_trainer
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/outputs/2026-02-01/20-42-57
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

code/RL_model/verl/Search-R1/outputs/2026-02-01/20-42-57/.hydra/overrides.yaml

@@ -0,0 +1,35 @@
+- data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet
+- data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet
+- data.train_batch_size=128
+- data.val_batch_size=64
+- data.max_prompt_length=4096
+- data.max_response_length=1024
+- data.shuffle_train_dataloader=True
+- algorithm.adv_estimator=grpo
+- actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507
+- actor_rollout_ref.model.enable_gradient_checkpointing=true
+- actor_rollout_ref.model.use_remove_padding=False
+- actor_rollout_ref.actor.optim.lr=1e-6
+- actor_rollout_ref.actor.ppo_mini_batch_size=64
+- +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16
+- actor_rollout_ref.actor.fsdp_config.param_offload=true
+- actor_rollout_ref.actor.fsdp_config.optimizer_offload=true
+- actor_rollout_ref.rollout.log_prob_micro_batch_size=64
+- actor_rollout_ref.rollout.tensor_model_parallel_size=1
+- actor_rollout_ref.rollout.name=vllm
+- actor_rollout_ref.rollout.gpu_memory_utilization=0.4
+- actor_rollout_ref.ref.log_prob_micro_batch_size=64
+- actor_rollout_ref.ref.fsdp_config.param_offload=True
+- actor_rollout_ref.actor.kl_loss_coef=0.001
+- trainer.logger=[wandb]
+- trainer.n_gpus_per_node=2
+- trainer.nnodes=1
+- trainer.save_freq=100
+- trainer.test_freq=50
+- trainer.project_name=
+- trainer.experiment_name=llm_guard_3B_10k_v2
+- trainer.total_epochs=15
+- trainer.total_training_steps=1005
+- trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2
+- do_search=false
+- max_turns=1

code/RL_model/verl/Search-R1/search_r1/llm_agent/generation.py

@@ -0,0 +1,469 @@
+import torch
+import re
+from collections import defaultdict
+import os
+from typing import List, Dict, Any, Tuple
+from dataclasses import dataclass
+from .tensor_helper import TensorHelper, TensorConfig
+from verl import DataProto
+from verl.utils.tracking import Tracking
+import shutil
+import requests
+
+@dataclass
+class GenerationConfig:
+    max_turns: int
+    max_start_length: int
+    max_prompt_length: int 
+    max_response_length: int
+    max_obs_length: int
+    num_gpus: int
+    no_think_rl: bool=False
+    search_url: str = None
+    topk: int = 3
+
+class LLMGenerationManager:
+    def __init__(
+        self,
+        tokenizer,
+        actor_rollout_wg,
+        config: GenerationConfig,
+        is_validation: bool = False,
+    ):
+        self.tokenizer = tokenizer
+        self.actor_rollout_wg = actor_rollout_wg
+        self.config = config
+        self.is_validation = is_validation
+
+        self.tensor_fn = TensorHelper(TensorConfig(
+            pad_token_id=tokenizer.pad_token_id,
+            max_prompt_length=config.max_prompt_length,
+            max_obs_length=config.max_obs_length,
+            max_start_length=config.max_start_length
+        ))
+
+    def _batch_tokenize(self, responses: List[str]) -> torch.Tensor:
+        """Tokenize a batch of responses."""
+        return self.tokenizer(
+            responses, 
+            add_special_tokens=False, 
+            return_tensors='pt', 
+            padding="longest"
+        )['input_ids']
+
+    def _postprocess_responses(self, responses: torch.Tensor) -> torch.Tensor:
+        """Process responses to stop at search operation or answer operation."""
+        responses_str = self.tokenizer.batch_decode(
+            responses, 
+            skip_special_tokens=True
+        )
+
+        responses_str = [resp.split('</search>')[0] + '</search>'
+                 if '</search>' in resp 
+                 else resp.split('</answer>')[0] + '</answer>'
+                 if '</answer>' in resp 
+                 else resp
+                 for resp in responses_str]
+
+        if self.config.no_think_rl:
+            raise ValueError('stop')
+            # if no_think_rl is enabled, only keep action in the str
+            actions, _ = self.env.postprocess_predictions(responses_str)
+            responses_str=[f"<answer>{envs[idx].ACTION_LOOKUP[action]}</answer>" for idx, action in enumerate(actions)]
+            print("RESPONSES:", responses_str)
+        responses = self._batch_tokenize(responses_str)
+        return responses, responses_str
+
+    def _process_next_obs(self, next_obs: List[str]) -> torch.Tensor:
+        """Process next observations from environment."""
+        
+        next_obs_ids = self.tokenizer(
+            next_obs, 
+            padding='longest',
+            return_tensors='pt',
+            add_special_tokens=False,  # Prevents adding special tokens
+        )['input_ids']
+
+        if next_obs_ids.shape[1] > self.config.max_obs_length:
+            print(f"[WARNING] OBSERVATION TOO LONG, CONSIDER CHANGING YOUR CONFIG, {next_obs_ids.shape[1]} & {self.config.max_obs_length}")            
+            next_obs_ids = next_obs_ids[:, :self.config.max_obs_length]
+
+        return next_obs_ids
+
+    def _update_rolling_state(self, rollings: DataProto, cur_responses: torch.Tensor, 
+                            next_obs_ids: torch.Tensor) -> Dict:
+        """Update rolling state with new responses and observations."""
+        # Concatenate and handle padding        
+        new_input_ids = self.tensor_fn.concatenate_with_padding([
+            rollings.batch['input_ids'],
+            cur_responses,
+            next_obs_ids
+        ])
+        
+        # Create attention mask and position ids
+        new_attention_mask = self.tensor_fn.create_attention_mask(new_input_ids)
+        new_position_ids = self.tensor_fn.create_position_ids(new_attention_mask)
+
+        # Cut to appropriate length
+        effective_len = new_attention_mask.sum(dim=1).max()
+        max_len = min(self.config.max_prompt_length, effective_len)
+
+        new_rollings = DataProto.from_dict({
+            'input_ids': new_input_ids[:, -max_len:],
+            'position_ids': new_position_ids[:, -max_len:],
+            'attention_mask': new_attention_mask[:, -max_len:]
+        })
+        new_rollings.meta_info.update(rollings.meta_info)
+        
+        return new_rollings
+
+    def _info_masked_concatenate_with_padding(self, 
+                prompt: torch.Tensor, 
+                prompt_with_mask: torch.Tensor, 
+                response: torch.Tensor, 
+                info: torch.Tensor = None,
+                pad_to_left: bool = True
+            ) -> torch.Tensor:
+        """Concatenate tensors and handle padding. Additionally, create a mask (info_mask) to cover the information block if it exists."""
+        pad_id = self.tokenizer.pad_token_id
+        tensors = [prompt, response]
+        tensors_with_mask = [prompt_with_mask, response]
+        if info is not None:
+            tensors.append(info)
+            info_mask = torch.full(info.size(), pad_id, dtype=info.dtype, device=info.device) # information mask
+            tensors_with_mask.append(info_mask)
+        
+        concatenated = torch.cat(tensors, dim=1)
+        concatenated_with_info = torch.cat(tensors_with_mask, dim=1)
+        mask = concatenated != pad_id if pad_to_left else concatenated == pad_id
+        sorted_indices = mask.to(torch.int64).argsort(dim=1, stable=True)
+        padded_tensor = concatenated.gather(1, sorted_indices)
+        padded_tensor_with_info = concatenated_with_info.gather(1, sorted_indices)
+
+        return padded_tensor, padded_tensor_with_info
+
+    def _update_right_side(self, right_side: Dict, 
+                          cur_responses: torch.Tensor,
+                          next_obs_ids: torch.Tensor = None) -> Dict:
+        """Update right side state."""
+        if next_obs_ids != None:
+            responses, responses_with_info_mask = self._info_masked_concatenate_with_padding(
+                    right_side['responses'],
+                    right_side['responses_with_info_mask'],
+                    cur_responses,
+                    next_obs_ids, 
+                    pad_to_left=False
+                )
+        else:
+            responses, responses_with_info_mask = self._info_masked_concatenate_with_padding(
+                    right_side['responses'],
+                    right_side['responses_with_info_mask'],
+                    cur_responses,
+                    pad_to_left=False
+                )
+        effective_len = self.tensor_fn.create_attention_mask(responses).sum(dim=1).max()
+        max_len = min(self.config.max_prompt_length, effective_len)
+        
+        return {'responses': responses[:, :max_len], 'responses_with_info_mask': responses_with_info_mask[:, :max_len]}
+
+    def _generate_with_gpu_padding(self, active_batch: DataProto) -> DataProto:
+        """
+            Wrapper for generation that handles multi-GPU padding requirements.
+            if num_gpus <= 1, return self.actor_rollout_wg.generate_sequences(active_batch)
+            if active_batch size is not divisible by num_gpus, pad with first sequence
+            then remove padding from output
+        """
+        num_gpus = self.config.num_gpus
+        if num_gpus <= 1:
+            return self.actor_rollout_wg.generate_sequences(active_batch)
+            
+        batch_size = active_batch.batch['input_ids'].shape[0]
+        remainder = batch_size % num_gpus
+        
+        for key in active_batch.batch.keys():
+            active_batch.batch[key] = active_batch.batch[key].long()
+        if remainder == 0:
+            return self.actor_rollout_wg.generate_sequences(active_batch)
+        
+        # Add padding sequences
+        padding_size = num_gpus - remainder
+        padded_batch = {}
+        
+        for k, v in active_batch.batch.items():
+            # Use first sequence as padding template
+            pad_sequence = v[0:1].repeat(padding_size, *[1] * (len(v.shape) - 1))
+            padded_batch[k] = torch.cat([v, pad_sequence], dim=0)
+
+        padded_active_batch = DataProto.from_dict(padded_batch)
+        for key in padded_active_batch.batch.keys():
+            padded_active_batch.batch[key] = padded_active_batch.batch[key].long()
+
+        # Generate with padded batch
+        padded_output = self.actor_rollout_wg.generate_sequences(padded_active_batch)
+
+        # Remove padding from output
+        trimmed_batch = {k: v[:-padding_size] for k, v in padded_output.batch.items()}
+        
+        # Handle meta_info if present
+        if hasattr(padded_output, 'meta_info') and padded_output.meta_info:
+            trimmed_meta = {}
+            for k, v in padded_output.meta_info.items():
+                if isinstance(v, torch.Tensor):
+                    trimmed_meta[k] = v[:-padding_size]
+                else:
+                    trimmed_meta[k] = v
+            padded_output.meta_info = trimmed_meta
+            
+        padded_output.batch = trimmed_batch
+        return padded_output
+
+    def run_llm_loop(self, gen_batch, initial_input_ids: torch.Tensor) -> Tuple[Dict, Dict]:
+        """Run main LLM generation loop."""
+        
+        original_left_side = {'input_ids': initial_input_ids[:, -self.config.max_start_length:]}
+        original_right_side = {'responses': initial_input_ids[:, []], 'responses_with_info_mask': initial_input_ids[:, []]}
+        
+        active_mask = torch.ones(gen_batch.batch['input_ids'].shape[0], dtype=torch.bool)
+        turns_stats = torch.ones(gen_batch.batch['input_ids'].shape[0], dtype=torch.int)
+        valid_action_stats = torch.zeros(gen_batch.batch['input_ids'].shape[0], dtype=torch.int)
+        valid_search_stats = torch.zeros(gen_batch.batch['input_ids'].shape[0], dtype=torch.int)
+        active_num_list = [active_mask.sum().item()]
+        rollings = gen_batch
+
+        # Main generation loop
+        for step in range(self.config.max_turns):
+            if not active_mask.sum():
+                break
+            rollings.batch = self.tensor_fn.cut_to_effective_len(
+                rollings.batch,
+                keys=['input_ids', 'attention_mask', 'position_ids']
+            )
+            
+            # gen_output = self.actor_rollout_wg.generate_sequences(rollings)
+            rollings_active = DataProto.from_dict({
+                k: v[active_mask] for k, v in rollings.batch.items()
+            })            
+            gen_output = self._generate_with_gpu_padding(rollings_active)
+
+            meta_info = gen_output.meta_info            
+            responses_ids, responses_str = self._postprocess_responses(gen_output.batch['responses'])
+            responses_ids, responses_str = self.tensor_fn._example_level_pad(responses_ids, responses_str, active_mask)
+
+            # Execute in environment and process observations
+            next_obs, dones, valid_action, is_search = self.execute_predictions(
+                responses_str, self.tokenizer.pad_token, active_mask
+            )
+            
+            curr_active_mask = torch.tensor([not done for done in dones], dtype=torch.bool)
+            active_mask = active_mask * curr_active_mask
+            active_num_list.append(active_mask.sum().item())
+            turns_stats[curr_active_mask] += 1
+            valid_action_stats += torch.tensor(valid_action, dtype=torch.int)
+            valid_search_stats += torch.tensor(is_search, dtype=torch.int)
+
+            next_obs_ids = self._process_next_obs(next_obs)
+            
+            # Update states
+            rollings = self._update_rolling_state(
+                rollings,
+                responses_ids,
+                next_obs_ids
+            )
+            original_right_side = self._update_right_side(
+                original_right_side,
+                responses_ids,
+                next_obs_ids
+            )
+            
+        # final LLM rollout
+        if active_mask.sum():
+            rollings.batch = self.tensor_fn.cut_to_effective_len(
+                rollings.batch,
+                keys=['input_ids', 'attention_mask', 'position_ids']
+            )
+
+            # gen_output = self.actor_rollout_wg.generate_sequences(rollings)
+            rollings_active = DataProto.from_dict({
+                k: v[active_mask] for k, v in rollings.batch.items()
+            })            
+            gen_output = self._generate_with_gpu_padding(rollings_active)
+
+            meta_info = gen_output.meta_info            
+            responses_ids, responses_str = self._postprocess_responses(gen_output.batch['responses'])
+            responses_ids, responses_str = self.tensor_fn._example_level_pad(responses_ids, responses_str, active_mask)
+
+            # # Execute in environment and process observations
+            _, dones, valid_action, is_search = self.execute_predictions(
+                responses_str, self.tokenizer.pad_token, active_mask, do_search=False
+            )
+
+            curr_active_mask = torch.tensor([not done for done in dones], dtype=torch.bool)
+            active_mask = active_mask * curr_active_mask
+            active_num_list.append(active_mask.sum().item())
+            valid_action_stats += torch.tensor(valid_action, dtype=torch.int)
+            valid_search_stats += torch.tensor(is_search, dtype=torch.int)
+            
+
+            original_right_side = self._update_right_side(
+                original_right_side,
+                responses_ids,
+            )
+        
+        meta_info['turns_stats'] = turns_stats.tolist()
+        meta_info['active_mask'] = active_mask.tolist()
+        meta_info['valid_action_stats'] = valid_action_stats.tolist()
+        meta_info['valid_search_stats'] = valid_search_stats.tolist()
+        
+        print("ACTIVE_TRAJ_NUM:", active_num_list)
+        
+        return self._compose_final_output(original_left_side, original_right_side, meta_info)
+
+    def _compose_final_output(self, left_side: Dict,
+                            right_side: Dict,
+                            meta_info: Dict) -> Tuple[Dict, Dict]:
+        """Compose final generation output."""
+        final_output = right_side.copy()
+        final_output['prompts'] = left_side['input_ids']
+        
+        # Combine input IDs
+        final_output['input_ids'] = torch.cat([
+            left_side['input_ids'],
+            right_side['responses']
+        ], dim=1)
+        
+        # Create attention mask and position ids
+        final_output['attention_mask'] = torch.cat([
+            self.tensor_fn.create_attention_mask(left_side['input_ids']),
+            self.tensor_fn.create_attention_mask(final_output['responses'])
+        ], dim=1)
+        final_output['info_mask'] = torch.cat([
+            self.tensor_fn.create_attention_mask(left_side['input_ids']),
+            self.tensor_fn.create_attention_mask(final_output['responses_with_info_mask'])
+        ], dim=1)
+        
+        final_output['position_ids'] = self.tensor_fn.create_position_ids(
+            final_output['attention_mask']
+        )
+        
+        final_output = DataProto.from_dict(final_output)
+        final_output.meta_info.update(meta_info)
+        
+        return final_output
+
+    def execute_predictions(self, predictions: List[str], pad_token: str, active_mask=None, do_search=True) -> List[str]:
+        """
+        Execute predictions across multiple environments.
+        NOTE: the function is the actual `step` function in the environment
+        NOTE penalty_for_invalid is not included in observation shown to the LLM
+        
+        Args:
+            envs: List of environment instances
+            predictions: List of action predictions
+            pad_token: Token to use for padding
+            
+        Returns:
+            List of observation strings
+        """
+        cur_actions, contents = self.postprocess_predictions(predictions)
+        next_obs, dones, valid_action, is_search = [], [], [], []
+        
+        search_queries = [content for action, content in zip(cur_actions, contents) if action == 'search']
+        if do_search:
+            search_results = self.batch_search(search_queries)
+            assert len(search_results) == sum([1 for action in cur_actions if action == 'search'])
+        else:
+            search_results = [''] * sum([1 for action in cur_actions if action == 'search'])
+
+        for i, (action, active) in enumerate(zip(cur_actions, active_mask)):
+            
+            if not active:
+                next_obs.append('')
+                dones.append(1)
+                valid_action.append(0)
+                is_search.append(0)
+            else:
+                if action == 'answer':
+                    next_obs.append('')
+                    dones.append(1)
+                    valid_action.append(1)
+                    is_search.append(0)
+                elif action == 'search':
+                    next_obs.append(f'\n\n<information>{search_results.pop(0).strip()}</information>\n\n')
+                    dones.append(0)
+                    valid_action.append(1)
+                    is_search.append(1)
+                else:
+                    next_obs.append(f'\nMy previous action is invalid. \
+If I want to search, I should put the query between <search> and </search>. \
+If I want to give the final answer, I should put the answer between <answer> and </answer>. Let me try again.\n')
+                    dones.append(0)
+                    valid_action.append(0)
+                    is_search.append(0)
+            
+        assert len(search_results) == 0
+            
+        return next_obs, dones, valid_action, is_search
+
+    def postprocess_predictions(self, predictions: List[Any]) -> Tuple[List[int], List[bool]]:
+        """
+        Process (text-based) predictions from llm into actions and validity flags.
+        
+        Args:
+            predictions: List of raw predictions
+            
+        Returns:
+            Tuple of (actions list, validity flags list)
+        """
+        actions = []
+        contents = []
+                
+        for prediction in predictions:
+            if isinstance(prediction, str): # for llm output
+                pattern = r'<(search|answer)>(.*?)</\1>'
+                match = re.search(pattern, prediction, re.DOTALL)
+                if match:
+                    content = match.group(2).strip()  # Return only the content inside the tags
+                    action = match.group(1)
+                else:
+                    content = ''
+                    action = None
+            else:
+                raise ValueError(f"Invalid prediction type: {type(prediction)}")
+            
+            actions.append(action)
+            contents.append(content)
+            
+        return actions, contents
+
+    def batch_search(self, queries: List[str] = None) -> str:
+        """
+        Batchified search for queries.
+        Args:
+            queries: queries to call the search engine
+        Returns:
+            search results which is concatenated into a string
+        """
+        results = self._batch_search(queries)['result']
+        
+        return [self._passages2string(result) for result in results]
+
+    def _batch_search(self, queries):
+        
+        payload = {
+            "queries": queries,
+            "topk": self.config.topk,
+            "return_scores": True
+        }
+        
+        return requests.post(self.config.search_url, json=payload).json()
+
+    def _passages2string(self, retrieval_result):
+        format_reference = ''
+        for idx, doc_item in enumerate(retrieval_result):
+            
+            content = doc_item['document']['contents']
+            title = content.split("\n")[0]
+            text = "\n".join(content.split("\n")[1:])
+            format_reference += f"Doc {idx+1}(Title: {title}) {text}\n"
+
+        return format_reference

code/RL_model/verl/Search-R1/search_r1/llm_agent/tensor_helper.py

@@ -0,0 +1,75 @@
+import torch
+from typing import Dict, Tuple, List
+from dataclasses import dataclass
+
+@dataclass
+class TensorConfig:
+    pad_token_id: int
+    max_prompt_length: int
+    max_obs_length: int
+    max_start_length: int
+
+class TensorHelper:
+    def __init__(self, config: TensorConfig):
+        self.config = config
+
+    def cut_to_effective_len(self, tensor_dict: Dict[str, torch.Tensor], 
+                            keys: List[str], cut_left: bool = True) -> Dict[str, torch.Tensor]:
+        """Cut tensors to their effective length based on attention mask."""
+        effective_len = tensor_dict['attention_mask'].sum(dim=1).max()
+        result = tensor_dict.copy()
+        
+        for key in keys:
+            if cut_left:
+                result[key] = tensor_dict[key][:, -effective_len:]
+            else:
+                result[key] = tensor_dict[key][:, :effective_len]
+        return result
+
+    def convert_pad_structure(self, tensor: torch.Tensor, pad_to_left: bool = True) -> Tuple[torch.Tensor, torch.Tensor]:
+        """Convert padding structure and return sorted tensor with indices."""
+        mask = tensor != self.config.pad_token_id if pad_to_left else tensor == self.config.pad_token_id
+        sorted_indices = mask.to(torch.int64).argsort(dim=1, stable=True)
+        return tensor.gather(1, sorted_indices), sorted_indices
+
+    def create_attention_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
+        """Create attention mask from input ids."""
+        return torch.where(input_ids != self.config.pad_token_id, 1, 0)
+
+    def create_position_ids(self, attention_mask: torch.Tensor) -> torch.Tensor:
+        """Create position ids from attention mask."""
+        return (torch.cumsum(attention_mask, dim=1) - 1) * attention_mask
+
+    def concatenate_with_padding(self, tensors: List[torch.Tensor], 
+                               pad_to_left: bool = True) -> torch.Tensor:
+        """Concatenate tensors and handle padding."""
+        concatenated = torch.cat(tensors, dim=1)
+        padded_tensor, _ = self.convert_pad_structure(concatenated, pad_to_left)
+        return padded_tensor
+
+    def _example_level_pad(self, responses: torch.Tensor, 
+                          responses_str: List[str], 
+                          active_mask: torch.Tensor) -> Tuple[torch.Tensor, List[str]]:
+        """
+        Pad responses for non-active examples with pad tokens.
+        """
+        assert active_mask.sum() == responses.shape[0]
+        # Create masked responses tensor
+        batch_size = active_mask.shape[0]
+        seq_len = responses.shape[1]
+        padded_responses = torch.full(
+            (batch_size, seq_len), self.config.pad_token_id,
+            dtype=responses.dtype, device=responses.device
+        )
+        padded_responses[active_mask] = responses
+        
+        # Create masked response strings
+        padded_responses_str = [""] * batch_size
+        
+        s = 0
+        for i, is_active in enumerate(active_mask):
+            if is_active:
+                padded_responses_str[i] = responses_str[s]
+                s += 1
+                
+        return padded_responses, padded_responses_str

code/RL_model/verl/Search-R1/search_r1/search/build_index.sh

@@ -0,0 +1,19 @@
+
+corpus_file=/your/corpus/jsonl/file # jsonl
+save_dir=/the/path/to/save/index
+retriever_name=e5 # this is for indexing naming
+retriever_model=intfloat/e5-base-v2
+
+# change faiss_type to HNSW32/64/128 for ANN indexing
+# change retriever_name to bm25 for BM25 indexing
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python index_builder.py \
+    --retrieval_method $retriever_name \
+    --model_path $retriever_model \
+    --corpus_path $corpus_file \
+    --save_dir $save_dir \
+    --use_fp16 \
+    --max_length 256 \
+    --batch_size 512 \
+    --pooling_method mean \
+    --faiss_type Flat \
+    --save_embedding

code/RL_model/verl/Search-R1/search_r1/search/google_search_server.py

@@ -0,0 +1,202 @@
+import os
+import re
+import requests
+import argparse
+import asyncio
+import random
+from typing import List, Optional, Dict
+from concurrent.futures import ThreadPoolExecutor
+
+import chardet
+import aiohttp
+import bs4
+import uvicorn
+from fastapi import FastAPI
+from pydantic import BaseModel
+from googleapiclient.discovery import build
+
+
+# --- CLI Args ---
+parser = argparse.ArgumentParser(description="Launch online search server.")
+parser.add_argument('--api_key', type=str, required=True, help="API key for Google search")
+parser.add_argument('--cse_id', type=str, required=True, help="CSE ID for Google search")
+parser.add_argument('--topk', type=int, default=3, help="Number of results to return per query")
+parser.add_argument('--snippet_only', action='store_true', help="If set, only return snippets; otherwise, return full context.")
+args = parser.parse_args()
+
+
+# --- Config ---
+class OnlineSearchConfig:
+    def __init__(self, topk: int = 3, api_key: Optional[str] = None, cse_id: Optional[str] = None, snippet_only: bool = False):
+        self.topk = topk
+        self.api_key = api_key
+        self.cse_id = cse_id
+        self.snippet_only = snippet_only
+
+
+# --- Utilities ---
+def parse_snippet(snippet: str) -> List[str]:
+    segments = snippet.split("...")
+    return [s.strip() for s in segments if len(s.strip().split()) > 5]
+
+
+def sanitize_search_query(query: str) -> str:
+    # Remove or replace special characters that might cause issues.
+    # This is a basic example; you might need to add more characters or patterns.
+    sanitized_query = re.sub(r'[^\w\s]', ' ', query)  # Replace non-alphanumeric and non-whitespace with spaces.
+    sanitized_query = re.sub(r'[\t\r\f\v\n]', ' ', sanitized_query) # replace tab, return, formfeed, vertical tab with spaces.
+    sanitized_query = re.sub(r'\s+', ' ', sanitized_query).strip() #remove duplicate spaces, and trailing/leading spaces.
+
+    return sanitized_query
+
+
+def filter_links(search_results: List[Dict]) -> List[str]:
+    links = []
+    for result in search_results:
+        for item in result.get("items", []):
+            if "mime" in item:
+                continue
+            ext = os.path.splitext(item["link"])[1]
+            if ext in ["", ".html", ".htm", ".shtml"]:
+                links.append(item["link"])
+    return links
+
+
+async def fetch(session: aiohttp.ClientSession, url: str, semaphore: asyncio.Semaphore) -> str:
+    user_agents = [
+        "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P)...",
+        "Mozilla/5.0 AppleWebKit/537.36...",
+        "Mozilla/5.0 (compatible; Googlebot/2.1; +https://www.google.com/bot.html)",
+    ]
+    headers = {"User-Agent": random.choice(user_agents)}
+
+    async with semaphore:
+        try:
+            async with session.get(url, headers=headers) as response:
+                raw = await response.read()
+                detected = chardet.detect(raw)
+                encoding = detected["encoding"] or "utf-8"
+                return raw.decode(encoding, errors="ignore")
+        except (aiohttp.ClientError, asyncio.TimeoutError):
+            return ""
+
+
+async def fetch_all(urls: List[str], limit: int = 8) -> List[str]:
+    semaphore = asyncio.Semaphore(limit)
+    timeout = aiohttp.ClientTimeout(total=5)
+    connector = aiohttp.TCPConnector(limit_per_host=limit, force_close=True)
+
+    async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
+        tasks = [fetch(session, url, semaphore) for url in urls]
+        return await asyncio.gather(*tasks)
+
+
+# --- Search Engine ---
+class OnlineSearchEngine:
+    def __init__(self, config: OnlineSearchConfig):
+        self.config = config
+
+    def collect_context(self, snippet: str, doc: str) -> str:
+        snippets = parse_snippet(snippet)
+        ctx_paras = []
+
+        for s in snippets:
+            pos = doc.replace("\n", " ").find(s)
+            if pos == -1:
+                continue
+            sta = pos
+            while sta > 0 and doc[sta] != "\n":
+                sta -= 1
+            end = pos + len(s)
+            while end < len(doc) and doc[end] != "\n":
+                end += 1
+            para = doc[sta:end].strip()
+            if para not in ctx_paras:
+                ctx_paras.append(para)
+
+        return "\n".join(ctx_paras)
+
+    def fetch_web_content(self, search_results: List[Dict]) -> Dict[str, str]:
+        links = filter_links(search_results)
+        contents = asyncio.run(fetch_all(links))
+        content_dict = {}
+        for html, link in zip(contents, links):
+            soup = bs4.BeautifulSoup(html, "html.parser")
+            text = "\n".join([p.get_text() for p in soup.find_all("p")])
+            content_dict[link] = text
+        return content_dict
+
+    def search(self, search_term: str, num_iter: int = 1) -> List[Dict]:
+        service = build('customsearch', 'v1', developerKey=self.config.api_key)
+        results = []
+        sanitize_search_term = sanitize_search_query(search_term)
+        if search_term.isspace():
+            return results
+        res = service.cse().list(q=sanitize_search_term, cx=self.config.cse_id).execute()
+        results.append(res)
+
+        for _ in range(num_iter - 1):
+            if 'nextPage' not in res.get('queries', {}):
+                break
+            start_idx = res['queries']['nextPage'][0]['startIndex']
+            res = service.cse().list(q=search_term, cx=self.config.cse_id, start=start_idx).execute()
+            results.append(res)
+
+        return results
+
+    def batch_search(self, queries: List[str]) -> List[List[str]]:
+        with ThreadPoolExecutor() as executor:
+            return list(executor.map(self._retrieve_context, queries))
+
+    def _retrieve_context(self, query: str) -> List[str]:
+        
+        if self.config.snippet_only:
+            search_results = self.search(query)
+            contexts = []
+            for result in search_results:
+                for item in result.get("items", []):
+                    title = item.get("title", "")
+                    context = ' '.join(parse_snippet(item.get("snippet", "")))
+                    if title != "" or context != "":
+                        title = "No title." if not title else title
+                        context = "No snippet available." if not context else context
+                        contexts.append({
+                            'document': {"contents": f'\"{title}\"\n{context}'},
+                        })
+        else:
+            content_dict = self.fetch_web_content(search_results)
+            contexts = []
+            for result in search_results:
+                for item in result.get("items", []):
+                    link = item["link"]
+                    title = item.get("title", "")
+                    snippet = item.get("snippet", "")
+                    if link in content_dict:
+                        context = self.collect_context(snippet, content_dict[link])
+                        if title != "" or context != "":
+                            title = "No title." if not title else title
+                            context = "No snippet available." if not context else context
+                            contexts.append({
+                                'document': {"contents": f'\"{title}\"\n{context}'},
+                            })
+        
+        return contexts[:self.config.topk]
+
+
+# --- FastAPI App ---
+app = FastAPI(title="Online Search Proxy Server")
+
+class SearchRequest(BaseModel):
+    queries: List[str]
+
+config = OnlineSearchConfig(api_key=args.api_key, cse_id=args.cse_id, topk=args.topk, snippet_only=args.snippet_only)
+engine = OnlineSearchEngine(config)
+
+@app.post("/retrieve")
+def search_endpoint(request: SearchRequest):
+    results = engine.batch_search(request.queries)
+    return {"result": results}
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

code/RL_model/verl/Search-R1/search_r1/search/index_builder.py

@@ -0,0 +1,349 @@
+import os
+import faiss
+import json
+import warnings
+import numpy as np
+from typing import cast, List, Dict
+import shutil
+import subprocess
+import argparse
+import torch
+from tqdm import tqdm
+# from LongRAG.retriever.utils import load_model, load_corpus, pooling
+import datasets
+from transformers import AutoTokenizer, AutoModel, AutoConfig
+
+
+def load_model(
+        model_path: str, 
+        use_fp16: bool = False
+    ):
+    model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+    model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
+    model.eval()
+    model.cuda()
+    if use_fp16: 
+        model = model.half()
+    tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
+
+    return model, tokenizer
+
+
+def pooling(
+        pooler_output,
+        last_hidden_state,
+        attention_mask = None,
+        pooling_method = "mean"
+    ):
+    if pooling_method == "mean":
+        last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
+        return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
+    elif pooling_method == "cls":
+        return last_hidden_state[:, 0]
+    elif pooling_method == "pooler":
+        return pooler_output
+    else:
+        raise NotImplementedError("Pooling method not implemented!")
+
+
+def load_corpus(corpus_path: str):
+    corpus = datasets.load_dataset(
+            'json', 
+            data_files=corpus_path,
+            split="train",
+            num_proc=4)
+    return corpus
+
+
+class Index_Builder:
+    r"""A tool class used to build an index used in retrieval.
+    
+    """
+    def __init__(
+            self, 
+            retrieval_method,
+            model_path,
+            corpus_path,
+            save_dir,
+            max_length,
+            batch_size,
+            use_fp16,
+            pooling_method,
+            faiss_type=None,
+            embedding_path=None,
+            save_embedding=False,
+            faiss_gpu=False
+        ):
+        
+        self.retrieval_method = retrieval_method.lower()
+        self.model_path = model_path
+        self.corpus_path = corpus_path
+        self.save_dir = save_dir
+        self.max_length = max_length
+        self.batch_size = batch_size
+        self.use_fp16 = use_fp16
+        self.pooling_method = pooling_method
+        self.faiss_type = faiss_type if faiss_type is not None else 'Flat'
+        self.embedding_path = embedding_path
+        self.save_embedding = save_embedding
+        self.faiss_gpu = faiss_gpu
+
+        self.gpu_num = torch.cuda.device_count()
+        # prepare save dir
+        print(self.save_dir)
+        if not os.path.exists(self.save_dir):
+            os.makedirs(self.save_dir)
+        else:
+            if not self._check_dir(self.save_dir):
+                warnings.warn("Some files already exists in save dir and may be overwritten.", UserWarning)
+
+        self.index_save_path = os.path.join(self.save_dir, f"{self.retrieval_method}_{self.faiss_type}.index")
+
+        self.embedding_save_path = os.path.join(self.save_dir, f"emb_{self.retrieval_method}.memmap")
+
+        self.corpus = load_corpus(self.corpus_path)
+       
+        print("Finish loading...")
+    @staticmethod
+    def _check_dir(dir_path):
+        r"""Check if the dir path exists and if there is content.
+        
+        """
+        
+        if os.path.isdir(dir_path):
+            if len(os.listdir(dir_path)) > 0:
+                return False
+        else:
+            os.makedirs(dir_path, exist_ok=True)
+        return True
+
+    def build_index(self):
+        r"""Constructing different indexes based on selective retrieval method.
+
+        """
+        if self.retrieval_method == "bm25":
+            self.build_bm25_index()
+        else:
+            self.build_dense_index()
+
+    def build_bm25_index(self):
+        """Building BM25 index based on Pyserini library.
+
+        Reference: https://github.com/castorini/pyserini/blob/master/docs/usage-index.md#building-a-bm25-index-direct-java-implementation
+        """
+
+        # to use pyserini pipeline, we first need to place jsonl file in the folder 
+        self.save_dir = os.path.join(self.save_dir, "bm25")
+        os.makedirs(self.save_dir, exist_ok=True)
+        temp_dir = self.save_dir + "/temp"
+        temp_file_path = temp_dir + "/temp.jsonl"
+        os.makedirs(temp_dir)
+
+        # if self.have_contents:
+        #     shutil.copyfile(self.corpus_path, temp_file_path)
+        # else:
+        #     with open(temp_file_path, "w") as f:
+        #         for item in self.corpus:
+        #             f.write(json.dumps(item) + "\n")
+        shutil.copyfile(self.corpus_path, temp_file_path)
+        
+        print("Start building bm25 index...")
+        pyserini_args = ["--collection", "JsonCollection",
+                         "--input", temp_dir,
+                         "--index", self.save_dir,
+                         "--generator", "DefaultLuceneDocumentGenerator",
+                         "--threads", "1"]
+       
+        subprocess.run(["python", "-m", "pyserini.index.lucene"] + pyserini_args)
+
+        shutil.rmtree(temp_dir)
+        
+        print("Finish!")
+
+    def _load_embedding(self, embedding_path, corpus_size, hidden_size):
+        all_embeddings = np.memmap(
+                embedding_path,
+                mode="r",
+                dtype=np.float32
+            ).reshape(corpus_size, hidden_size)
+        return all_embeddings
+
+    def _save_embedding(self, all_embeddings):
+        memmap = np.memmap(
+            self.embedding_save_path,
+            shape=all_embeddings.shape,
+            mode="w+",
+            dtype=all_embeddings.dtype
+        )
+        length = all_embeddings.shape[0]
+        # add in batch
+        save_batch_size = 10000
+        if length > save_batch_size:
+            for i in tqdm(range(0, length, save_batch_size), leave=False, desc="Saving Embeddings"):
+                j = min(i + save_batch_size, length)
+                memmap[i: j] = all_embeddings[i: j]
+        else:
+            memmap[:] = all_embeddings
+
+    def encode_all(self):
+        if self.gpu_num > 1:
+            print("Use multi gpu!")
+            self.encoder = torch.nn.DataParallel(self.encoder)
+            self.batch_size = self.batch_size * self.gpu_num
+
+        all_embeddings = []
+
+        for start_idx in tqdm(range(0, len(self.corpus), self.batch_size), desc='Inference Embeddings:'):
+
+            # batch_data_title = self.corpus[start_idx:start_idx+self.batch_size]['title']
+            # batch_data_text = self.corpus[start_idx:start_idx+self.batch_size]['text']
+            # batch_data = ['"' + title + '"\n' + text for title, text in zip(batch_data_title, batch_data_text)]
+            batch_data = self.corpus[start_idx:start_idx+self.batch_size]['contents']
+
+            if self.retrieval_method == "e5":
+                batch_data = [f"passage: {doc}" for doc in batch_data]
+
+            inputs = self.tokenizer(
+                        batch_data,
+                        padding=True,
+                        truncation=True,
+                        return_tensors='pt',
+                        max_length=self.max_length,
+            ).to('cuda')
+
+            inputs = {k: v.cuda() for k, v in inputs.items()}
+
+            #TODO: support encoder-only T5 model
+            if "T5" in type(self.encoder).__name__:
+                # T5-based retrieval model
+                decoder_input_ids = torch.zeros(
+                    (inputs['input_ids'].shape[0], 1), dtype=torch.long
+                ).to(inputs['input_ids'].device)
+                output = self.encoder(
+                    **inputs, decoder_input_ids=decoder_input_ids, return_dict=True
+                )
+                embeddings = output.last_hidden_state[:, 0, :]
+
+            else:
+                output = self.encoder(**inputs, return_dict=True)
+                embeddings = pooling(output.pooler_output, 
+                                    output.last_hidden_state, 
+                                    inputs['attention_mask'],
+                                    self.pooling_method)
+                if  "dpr" not in self.retrieval_method:
+                    embeddings = torch.nn.functional.normalize(embeddings, dim=-1)
+
+            embeddings = cast(torch.Tensor, embeddings)
+            embeddings = embeddings.detach().cpu().numpy()
+            all_embeddings.append(embeddings)
+
+        all_embeddings = np.concatenate(all_embeddings, axis=0)
+        all_embeddings = all_embeddings.astype(np.float32)
+
+        return all_embeddings
+
+    @torch.no_grad()
+    def build_dense_index(self):
+        """Obtain the representation of documents based on the embedding model(BERT-based) and 
+        construct a faiss index.
+        """
+        
+        if os.path.exists(self.index_save_path):
+            print("The index file already exists and will be overwritten.")
+        
+        self.encoder, self.tokenizer = load_model(model_path = self.model_path, 
+                                                  use_fp16 = self.use_fp16)
+        if self.embedding_path is not None:
+            hidden_size = self.encoder.config.hidden_size
+            corpus_size = len(self.corpus)
+            all_embeddings = self._load_embedding(self.embedding_path, corpus_size, hidden_size)
+        else:
+            all_embeddings = self.encode_all()
+            if self.save_embedding:
+                self._save_embedding(all_embeddings)
+            del self.corpus
+
+        # build index
+        print("Creating index")
+        dim = all_embeddings.shape[-1]
+        faiss_index = faiss.index_factory(dim, self.faiss_type, faiss.METRIC_INNER_PRODUCT)
+        
+        if self.faiss_gpu:
+            co = faiss.GpuMultipleClonerOptions()
+            co.useFloat16 = True
+            co.shard = True
+            faiss_index = faiss.index_cpu_to_all_gpus(faiss_index, co)
+            if not faiss_index.is_trained:
+                faiss_index.train(all_embeddings)
+            faiss_index.add(all_embeddings)
+            faiss_index = faiss.index_gpu_to_cpu(faiss_index)
+        else:
+            if not faiss_index.is_trained:
+                faiss_index.train(all_embeddings)
+            faiss_index.add(all_embeddings)
+
+        faiss.write_index(faiss_index, self.index_save_path)
+        print("Finish!")
+
+
+MODEL2POOLING = {
+    "e5": "mean",
+    "bge": "cls",
+    "contriever": "mean",
+    'jina': 'mean'
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description = "Creating index.")
+
+    # Basic parameters
+    parser.add_argument('--retrieval_method', type=str)
+    parser.add_argument('--model_path', type=str, default=None)
+    parser.add_argument('--corpus_path', type=str)
+    parser.add_argument('--save_dir', default= 'indexes/',type=str)
+
+    # Parameters for building dense index
+    parser.add_argument('--max_length', type=int, default=180)
+    parser.add_argument('--batch_size', type=int, default=512)
+    parser.add_argument('--use_fp16', default=False, action='store_true')
+    parser.add_argument('--pooling_method', type=str, default=None)
+    parser.add_argument('--faiss_type',default=None,type=str)
+    parser.add_argument('--embedding_path', default=None, type=str)
+    parser.add_argument('--save_embedding', action='store_true', default=False)
+    parser.add_argument('--faiss_gpu', default=False, action='store_true')
+    
+    args = parser.parse_args()
+
+    if args.pooling_method is None:
+        pooling_method = 'mean'
+        for k,v in MODEL2POOLING.items():
+            if k in args.retrieval_method.lower():
+                pooling_method = v
+                break
+    else:
+        if args.pooling_method not in ['mean','cls','pooler']:
+            raise NotImplementedError
+        else:
+            pooling_method = args.pooling_method
+
+
+    index_builder = Index_Builder(
+                        retrieval_method = args.retrieval_method,
+                        model_path = args.model_path,
+                        corpus_path = args.corpus_path,
+                        save_dir = args.save_dir,
+                        max_length = args.max_length,
+                        batch_size = args.batch_size,
+                        use_fp16 = args.use_fp16,
+                        pooling_method = pooling_method,
+                        faiss_type = args.faiss_type,
+                        embedding_path = args.embedding_path,
+                        save_embedding = args.save_embedding,
+                        faiss_gpu = args.faiss_gpu
+                    )
+    index_builder.build_index()
+
+
+if __name__ == "__main__":
+    main()

code/RL_model/verl/Search-R1/search_r1/search/rerank_server.py

@@ -0,0 +1,161 @@
+import argparse
+from collections import defaultdict
+from typing import Optional
+from dataclasses import dataclass, field
+
+from sentence_transformers import CrossEncoder
+import torch
+from transformers import HfArgumentParser
+import numpy as np
+
+import uvicorn
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+
+class BaseCrossEncoder:
+    def __init__(self, model, batch_size=32, device="cuda"):
+        self.model = model
+        self.batch_size = batch_size
+        self.model.to(device)
+
+    def _passage_to_string(self, doc_item):
+        if "document" not in doc_item:
+            content = doc_item['contents']
+        else:
+            content = doc_item['document']['contents']
+        title = content.split("\n")[0]
+        text = "\n".join(content.split("\n")[1:])
+
+        return f"(Title: {title}) {text}"
+
+    def rerank(self, 
+               queries: list[str], 
+               documents: list[list[dict]]):
+        """
+        Assume documents is a list of list of dicts, where each dict is a document with keys "id" and "contents".
+        This asumption is made to be consistent with the output of the retrieval server.
+        """ 
+        assert len(queries) == len(documents)
+
+        pairs = []
+        qids = []
+        for qid, query in enumerate(queries):
+            for document in documents:
+                for doc_item in document:
+                    doc = self._passage_to_string(doc_item)
+                    pairs.append((query, doc))
+                    qids.append(qid)
+
+        scores = self._predict(pairs)
+        query_to_doc_scores = defaultdict(list)
+
+        assert len(scores) == len(pairs) == len(qids)
+        for i in range(len(pairs)):
+            query, doc = pairs[i]
+            score = scores[i] 
+            qid = qids[i]
+            query_to_doc_scores[qid].append((doc, score))
+
+        sorted_query_to_doc_scores = {}
+        for query, doc_scores in query_to_doc_scores.items():
+            sorted_query_to_doc_scores[query] = sorted(doc_scores, key=lambda x: x[1], reverse=True)
+
+        return sorted_query_to_doc_scores
+
+    def _predict(self, pairs: list[tuple[str, str]]):
+        raise NotImplementedError 
+
+    @classmethod
+    def load(cls, model_name_or_path, **kwargs):
+        raise NotImplementedError
+
+
+class SentenceTransformerCrossEncoder(BaseCrossEncoder):
+    def __init__(self, model, batch_size=32, device="cuda"):
+        super().__init__(model, batch_size, device)
+
+    def _predict(self, pairs: list[tuple[str, str]]):
+        scores = self.model.predict(pairs, batch_size=self.batch_size)
+        scores = scores.tolist() if isinstance(scores, torch.Tensor) or isinstance(scores, np.ndarray) else scores
+        return scores
+
+    @classmethod
+    def load(cls, model_name_or_path, **kwargs):
+        model = CrossEncoder(model_name_or_path)
+        return cls(model, **kwargs)
+
+
+class RerankRequest(BaseModel):
+    queries: list[str]
+    documents: list[list[dict]]
+    rerank_topk: Optional[int] = None
+    return_scores: bool = False
+
+
+@dataclass 
+class RerankerArguments:
+    max_length: int = field(default=512)
+    rerank_topk: int = field(default=3)
+    rerank_model_name_or_path: str = field(default="cross-encoder/ms-marco-MiniLM-L12-v2")
+    batch_size: int = field(default=32)
+    reranker_type: str = field(default="sentence_transformer")
+
+def get_reranker(config):
+    if config.reranker_type == "sentence_transformer":
+        return SentenceTransformerCrossEncoder.load(
+            config.rerank_model_name_or_path,
+            batch_size=config.batch_size,
+            device="cuda" if torch.cuda.is_available() else "cpu"
+        )
+    else:
+        raise ValueError(f"Unknown reranker type: {config.reranker_type}")
+
+
+app = FastAPI()
+
+@app.post("/rerank")
+def rerank_endpoint(request: RerankRequest):
+    """
+    Endpoint that accepts queries and performs retrieval.
+    Input format:
+    {
+      "queries": ["What is Python?", "Tell me about neural networks."],
+      "documents": [[doc_item_1, ..., doc_item_k], [doc_item_1, ..., doc_item_k]],
+      "rerank_topk": 3,
+      "return_scores": true
+    }
+    """
+    if not request.rerank_topk:
+        request.rerank_topk = config.rerank_topk  # fallback to default
+
+    # Perform batch re reranking
+    # doc_scores already sorted by score
+    query_to_doc_scores = reranker.rerank(request.queries, request.documents) 
+
+    # Format response 
+    resp = []
+    for _, doc_scores in query_to_doc_scores.items():
+        doc_scores = doc_scores[:request.rerank_topk]
+        if request.return_scores:
+            combined = [] 
+            for doc, score in doc_scores:
+                combined.append({"document": doc, "score": score})
+            resp.append(combined)
+        else:
+            resp.append([doc for doc, _ in doc_scores])
+    return {"result": resp}
+
+
+if __name__ == "__main__":
+    
+    # 1) Build a config (could also parse from arguments).
+    #    In real usage, you'd parse your CLI arguments or environment variables.
+    parser = HfArgumentParser((RerankerArguments))
+    config = parser.parse_args_into_dataclasses()[0]
+
+    # 2) Instantiate a global retriever so it is loaded once and reused.
+    reranker = get_reranker(config)
+    
+    # 3) Launch the server. By default, it listens on http://127.0.0.1:8000
+    uvicorn.run(app, host="0.0.0.0", port=6980)

code/RL_model/verl/Search-R1/search_r1/search/retrieval.py

@@ -0,0 +1,368 @@
+import json
+import os
+import warnings
+from typing import List, Dict
+import functools
+from tqdm import tqdm
+from multiprocessing import Pool
+import faiss
+import torch
+import numpy as np
+from transformers import AutoConfig, AutoTokenizer, AutoModel
+import argparse
+import datasets
+
+
+def load_corpus(corpus_path: str):
+    corpus = datasets.load_dataset(
+            'json', 
+            data_files=corpus_path,
+            split="train",
+            num_proc=4)
+    return corpus
+    
+
+def read_jsonl(file_path):
+    data = []
+    
+    with open(file_path, "r") as f:
+        readin = f.readlines()
+        for line in readin:
+            data.append(json.loads(line))
+    return data
+
+
+def load_docs(corpus, doc_idxs):
+    results = [corpus[int(idx)] for idx in doc_idxs]
+
+    return results
+
+
+def load_model(
+        model_path: str, 
+        use_fp16: bool = False
+    ):
+    model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+    model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
+    model.eval()
+    model.cuda()
+    if use_fp16: 
+        model = model.half()
+    tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
+
+    return model, tokenizer
+
+
+def pooling(
+        pooler_output,
+        last_hidden_state,
+        attention_mask = None,
+        pooling_method = "mean"
+    ):
+    if pooling_method == "mean":
+        last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
+        return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
+    elif pooling_method == "cls":
+        return last_hidden_state[:, 0]
+    elif pooling_method == "pooler":
+        return pooler_output
+    else:
+        raise NotImplementedError("Pooling method not implemented!")
+
+
+class Encoder:
+    def __init__(self, model_name, model_path, pooling_method, max_length, use_fp16):
+        self.model_name = model_name
+        self.model_path = model_path
+        self.pooling_method = pooling_method
+        self.max_length = max_length
+        self.use_fp16 = use_fp16
+
+        self.model, self.tokenizer = load_model(model_path=model_path,
+                                                use_fp16=use_fp16)
+
+    @torch.no_grad()
+    def encode(self, query_list: List[str], is_query=True) -> np.ndarray:
+        # processing query for different encoders
+        if isinstance(query_list, str):
+            query_list = [query_list]
+
+        if "e5" in self.model_name.lower():
+            if is_query:
+                query_list = [f"query: {query}" for query in query_list]
+            else:
+                query_list = [f"passage: {query}" for query in query_list]
+
+        if "bge" in self.model_name.lower():
+            if is_query:
+                query_list = [f"Represent this sentence for searching relevant passages: {query}" for query in query_list]
+
+        inputs = self.tokenizer(query_list,
+                                max_length=self.max_length,
+                                padding=True,
+                                truncation=True,
+                                return_tensors="pt"
+                                )
+        inputs = {k: v.cuda() for k, v in inputs.items()}
+
+        if "T5" in type(self.model).__name__:
+            # T5-based retrieval model
+            decoder_input_ids = torch.zeros(
+                (inputs['input_ids'].shape[0], 1), dtype=torch.long
+            ).to(inputs['input_ids'].device)
+            output = self.model(
+                **inputs, decoder_input_ids=decoder_input_ids, return_dict=True
+            )
+            query_emb = output.last_hidden_state[:, 0, :]
+
+        else:
+            output = self.model(**inputs, return_dict=True)
+            query_emb = pooling(output.pooler_output,
+                                output.last_hidden_state,
+                                inputs['attention_mask'],
+                                self.pooling_method)
+            if "dpr" not in self.model_name.lower():
+                query_emb = torch.nn.functional.normalize(query_emb, dim=-1)
+
+        query_emb = query_emb.detach().cpu().numpy()
+        query_emb = query_emb.astype(np.float32, order="C")
+        return query_emb
+
+
+class BaseRetriever:
+    """Base object for all retrievers."""
+
+    def __init__(self, config):
+        self.config = config
+        self.retrieval_method = config.retrieval_method
+        self.topk = config.retrieval_topk
+        
+        self.index_path = config.index_path
+        self.corpus_path = config.corpus_path
+
+        # self.cache_save_path = os.path.join(config.save_dir, 'retrieval_cache.json')
+
+    def _search(self, query: str, num: int, return_score:bool) -> List[Dict[str, str]]:
+        r"""Retrieve topk relevant documents in corpus.
+        Return:
+            list: contains information related to the document, including:
+                contents: used for building index
+                title: (if provided)
+                text: (if provided)
+        """
+        pass
+
+    def _batch_search(self, query_list, num, return_score):
+        pass
+
+    def search(self, *args, **kwargs):
+        return self._search(*args, **kwargs)
+    
+    def batch_search(self, *args, **kwargs):
+        return self._batch_search(*args, **kwargs)
+
+
+class BM25Retriever(BaseRetriever):
+    r"""BM25 retriever based on pre-built pyserini index."""
+
+    def __init__(self, config):
+        super().__init__(config)
+        from pyserini.search.lucene import LuceneSearcher
+        self.searcher = LuceneSearcher(self.index_path)
+        self.contain_doc = self._check_contain_doc()
+        if not self.contain_doc:
+            self.corpus = load_corpus(self.corpus_path)
+        self.max_process_num = 8
+        
+    def _check_contain_doc(self):
+        r"""Check if the index contains document content
+        """
+        return self.searcher.doc(0).raw() is not None
+
+    def _search(self, query: str, num: int = None, return_score = False) -> List[Dict[str, str]]:
+        if num is None:
+            num = self.topk
+        
+        hits = self.searcher.search(query, num)
+        if len(hits) < 1:
+            if return_score:
+                return [],[]
+            else:
+                return []
+            
+        scores = [hit.score for hit in hits]
+        if len(hits) < num:
+            warnings.warn('Not enough documents retrieved!')
+        else:
+            hits = hits[:num]
+
+        if self.contain_doc:
+            all_contents = [json.loads(self.searcher.doc(hit.docid).raw())['contents'] for hit in hits]
+            results = [{'title': content.split("\n")[0].strip("\""), 
+                        'text': "\n".join(content.split("\n")[1:]),
+                        'contents': content} for content in all_contents]
+        else:
+            results = load_docs(self.corpus, [hit.docid for hit in hits])
+
+        if return_score:
+            return results, scores
+        else:
+            return results
+
+    def _batch_search(self, query_list, num: int = None, return_score = False):
+        # TODO: modify batch method
+        results = []
+        scores = []
+        for query in query_list:
+            item_result, item_score = self._search(query, num,True)
+            results.append(item_result)
+            scores.append(item_score)
+
+        if return_score:
+            return results, scores
+        else:
+            return results
+
+def get_available_gpu_memory():
+    memory_info = []
+    for i in range(torch.cuda.device_count()):
+        total_memory = torch.cuda.get_device_properties(i).total_memory
+        allocated_memory = torch.cuda.memory_allocated(i)
+        free_memory = total_memory - allocated_memory
+        memory_info.append((i, free_memory / 1e9))  # Convert to GB
+    return memory_info
+
+
+class DenseRetriever(BaseRetriever):
+    r"""Dense retriever based on pre-built faiss index."""
+
+    def __init__(self, config: dict):
+        super().__init__(config)
+        self.index = faiss.read_index(self.index_path)
+        if config.faiss_gpu:
+            co = faiss.GpuMultipleClonerOptions()
+            co.useFloat16 = True
+            co.shard = True
+            self.index = faiss.index_cpu_to_all_gpus(self.index, co=co)
+            # self.index = faiss.index_cpu_to_all_gpus(self.index)
+
+        self.corpus = load_corpus(self.corpus_path)
+        self.encoder = Encoder(
+             model_name = self.retrieval_method, 
+             model_path = config.retrieval_model_path,
+             pooling_method = config.retrieval_pooling_method,
+             max_length = config.retrieval_query_max_length,
+             use_fp16 = config.retrieval_use_fp16
+            )
+        self.topk = config.retrieval_topk
+        self.batch_size = self.config.retrieval_batch_size
+
+    def _search(self, query: str, num: int = None, return_score = False):
+        if num is None:
+            num = self.topk
+        query_emb = self.encoder.encode(query)
+        scores, idxs = self.index.search(query_emb, k=num)
+        idxs = idxs[0]
+        scores = scores[0]
+
+        results = load_docs(self.corpus, idxs)
+        if return_score:
+            return results, scores
+        else:
+            return results
+
+    def _batch_search(self, query_list: List[str], num: int = None, return_score = False):
+        if isinstance(query_list, str):
+            query_list = [query_list]
+        if num is None:
+            num = self.topk
+        
+        batch_size = self.batch_size
+
+        results = []
+        scores = []
+
+        for start_idx in tqdm(range(0, len(query_list), batch_size), desc='Retrieval process: '):
+            query_batch = query_list[start_idx:start_idx + batch_size]
+            
+            # from time import time
+            # a = time()
+            batch_emb = self.encoder.encode(query_batch)
+            # b = time()
+            # print(f'################### encode time {b-a} #####################')
+            batch_scores, batch_idxs = self.index.search(batch_emb, k=num)
+            batch_scores = batch_scores.tolist()
+            batch_idxs = batch_idxs.tolist()
+            # print(f'################### search time {time()-b} #####################')
+            # exit()
+            
+            flat_idxs = sum(batch_idxs, [])
+            batch_results = load_docs(self.corpus, flat_idxs)
+            batch_results = [batch_results[i*num : (i+1)*num] for i in range(len(batch_idxs))]
+            
+            scores.extend(batch_scores)
+            results.extend(batch_results)
+        
+        if return_score:
+            return results, scores
+        else:
+            return results
+
+def get_retriever(config):
+    r"""Automatically select retriever class based on config's retrieval method
+
+    Args:
+        config (dict): configuration with 'retrieval_method' key
+
+    Returns:
+        Retriever: retriever instance
+    """
+    if config.retrieval_method == "bm25":
+        return BM25Retriever(config)
+    else:
+        return DenseRetriever(config)
+
+
+def get_dataset(config):
+    """Load dataset from config."""
+     
+    split_path = os.path.join(config.dataset_path, f'{config.data_split}.jsonl')
+    return read_jsonl(split_path)
+
+
+if __name__ == '__main__':
+    
+    parser = argparse.ArgumentParser(description = "Retrieval")
+
+    # Basic parameters
+    parser.add_argument('--retrieval_method', type=str)
+    parser.add_argument('--retrieval_topk', type=int, default=10)
+    parser.add_argument('--index_path', type=str, default=None)
+    parser.add_argument('--corpus_path', type=str)
+    parser.add_argument('--dataset_path', default=None, type=str)
+
+    parser.add_argument('--faiss_gpu', default=True, type=bool)
+    parser.add_argument('--data_split', default="train", type=str)
+    
+    parser.add_argument('--retrieval_model_path', type=str, default=None)
+    parser.add_argument('--retrieval_pooling_method', default='mean', type=str)
+    parser.add_argument('--retrieval_query_max_length', default=256, type=str)
+    parser.add_argument('--retrieval_use_fp16', action='store_true', default=False)
+    parser.add_argument('--retrieval_batch_size', default=512, type=int)
+    
+    args = parser.parse_args()
+
+    args.index_path = os.path.join(args.index_path, f'{args.retrieval_method}_Flat.index') if args.retrieval_method != 'bm25' else os.path.join(args.index_path, 'bm25')
+
+    # load dataset
+    all_split = get_dataset(args)
+    
+    input_query = [sample['question'] for sample in all_split[:512]]
+    
+    # initialize the retriever and conduct retrieval
+    retriever = get_retriever(args)
+    print('Start Retrieving ...')    
+    results, scores = retriever.batch_search(input_query, return_score=True)
+
+    # from IPython import embed
+    # embed()

code/RL_model/verl/Search-R1/search_r1/search/retrieval.sh

@@ -0,0 +1,25 @@
+
+DATA_NAME=nq
+
+DATASET_PATH="/home/peterjin/mnt/data/$DATA_NAME"
+
+SPLIT='test'
+TOPK=3
+
+INDEX_PATH=/home/peterjin/mnt/index/wiki-18
+CORPUS_PATH=/home/peterjin/mnt/data/retrieval-corpus/wiki-18.jsonl
+SAVE_NAME=e5_${TOPK}_wiki18.json
+
+# INDEX_PATH=/home/peterjin/rm_retrieval_corpus/index/wiki-21
+# CORPUS_PATH=/home/peterjin/rm_retrieval_corpus/corpora/wiki/enwiki-dec2021/text-list-100-sec.jsonl
+# SAVE_NAME=e5_${TOPK}_wiki21.json
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python retrieval.py --retrieval_method e5 \
+                    --retrieval_topk $TOPK \
+                    --index_path $INDEX_PATH \
+                    --corpus_path $CORPUS_PATH \
+                    --dataset_path $DATASET_PATH \
+                    --data_split $SPLIT \
+                    --retrieval_model_path "intfloat/e5-base-v2" \
+                    --retrieval_pooling_method "mean" \
+                    --retrieval_batch_size 512 \

code/RL_model/verl/Search-R1/search_r1/search/retrieval_request.py

@@ -0,0 +1,23 @@
+import requests
+
+# URL for your local FastAPI server
+url = "http://127.0.0.1:8000/retrieve"
+
+# Example payload
+payload = {
+    "queries": ["What is the capital of France?", "Explain neural networks."] * 200,
+    "topk": 5,
+    "return_scores": True
+}
+
+# Send POST request
+response = requests.post(url, json=payload)
+
+# Raise an exception if the request failed
+response.raise_for_status()
+
+# Get the JSON response
+retrieved_data = response.json()
+
+print("Response from server:")
+print(retrieved_data)

code/RL_model/verl/Search-R1/search_r1/search/retrieval_rerank_server.py

@@ -0,0 +1,123 @@
+# pip install -U sentence-transformers
+import os
+import re
+import argparse
+from dataclasses import dataclass, field
+from typing import List, Optional
+from collections import defaultdict
+
+import torch
+import numpy as np
+from fastapi import FastAPI
+from pydantic import BaseModel
+from sentence_transformers import CrossEncoder
+
+from retrieval_server import get_retriever, Config as RetrieverConfig
+from rerank_server import SentenceTransformerCrossEncoder
+
+app = FastAPI()
+
+def convert_title_format(text):
+    # Use regex to extract the title and the content
+    match = re.match(r'\(Title:\s*([^)]+)\)\s*(.+)', text, re.DOTALL)
+    if match:
+        title, content = match.groups()
+        return f'\"{title}\"\n{content}'
+    else:
+        return text
+
+# ----------- Combined Request Schema -----------
+class SearchRequest(BaseModel):
+    queries: List[str]
+    topk_retrieval: Optional[int] = 10
+    topk_rerank: Optional[int] = 3
+    return_scores: bool = False
+
+# ----------- Reranker Config Schema -----------
+@dataclass
+class RerankerArguments:
+    max_length: int = field(default=512)
+    rerank_topk: int = field(default=3)
+    rerank_model_name_or_path: str = field(default="cross-encoder/ms-marco-MiniLM-L12-v2")
+    batch_size: int = field(default=32)
+    reranker_type: str = field(default="sentence_transformer")
+
+def get_reranker(config):
+    if config.reranker_type == "sentence_transformer":
+        return SentenceTransformerCrossEncoder.load(
+            config.rerank_model_name_or_path,
+            batch_size=config.batch_size,
+            device="cuda" if torch.cuda.is_available() else "cpu"
+        )
+    else:
+        raise ValueError(f"Unknown reranker type: {config.reranker_type}")
+
+# ----------- Endpoint -----------
+@app.post("/retrieve")
+def search_endpoint(request: SearchRequest):
+    # Step 1: Retrieve documents
+    retrieved_docs = retriever.batch_search(
+        query_list=request.queries,
+        num=request.topk_retrieval,
+        return_score=False
+    )
+
+    # Step 2: Rerank
+    reranked = reranker.rerank(request.queries, retrieved_docs)
+
+    # Step 3: Format response
+    response = []
+    for i, doc_scores in reranked.items():
+        doc_scores = doc_scores[:request.topk_rerank]
+        if request.return_scores:
+            combined = []
+            for doc, score in doc_scores:
+                combined.append({"document": convert_title_format(doc), "score": score})
+            response.append(combined)
+        else:
+            response.append([convert_title_format(doc) for doc, _ in doc_scores])
+
+    return {"result": response}
+
+
+if __name__ == "__main__":
+    
+    parser = argparse.ArgumentParser(description="Launch the local faiss retriever.")
+    # retriever
+    parser.add_argument("--index_path", type=str, default="/home/peterjin/mnt/index/wiki-18/e5_Flat.index", help="Corpus indexing file.")
+    parser.add_argument("--corpus_path", type=str, default="/home/peterjin/mnt/data/retrieval-corpus/wiki-18.jsonl", help="Local corpus file.")
+    parser.add_argument("--retrieval_topk", type=int, default=10, help="Number of retrieved passages for one query.")
+    parser.add_argument("--retriever_name", type=str, default="e5", help="Name of the retriever model.")
+    parser.add_argument("--retriever_model", type=str, default="intfloat/e5-base-v2", help="Path of the retriever model.")
+    parser.add_argument('--faiss_gpu', action='store_true', help='Use GPU for computation')
+    # reranker
+    parser.add_argument("--reranking_topk", type=int, default=3, help="Number of reranked passages for one query.")
+    parser.add_argument("--reranker_model", type=str, default="cross-encoder/ms-marco-MiniLM-L12-v2", help="Path of the reranker model.")
+    parser.add_argument("--reranker_batch_size", type=int, default=32, help="Batch size for the reranker inference.")
+
+    args = parser.parse_args()
+    
+    # ----------- Load Retriever and Reranker -----------
+    retriever_config = RetrieverConfig(
+        retrieval_method = args.retriever_name,
+        index_path=args.index_path,
+        corpus_path=args.corpus_path,
+        retrieval_topk=args.retrieval_topk,
+        faiss_gpu=args.faiss_gpu,
+        retrieval_model_path=args.retriever_model,
+        retrieval_pooling_method="mean",
+        retrieval_query_max_length=256,
+        retrieval_use_fp16=True,
+        retrieval_batch_size=512,
+    )
+    retriever = get_retriever(retriever_config)
+
+    reranker_config = RerankerArguments(
+        rerank_topk = args.reranking_topk,
+        rerank_model_name_or_path = args.reranker_model,
+        batch_size = args.reranker_batch_size,
+    )
+    reranker = get_reranker(reranker_config)
+    
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

code/RL_model/verl/Search-R1/search_r1/search/retrieval_server.py

@@ -0,0 +1,392 @@
+import json
+import os
+import warnings
+from typing import List, Dict, Optional
+import argparse
+
+import faiss
+import torch
+import numpy as np
+from transformers import AutoConfig, AutoTokenizer, AutoModel
+from tqdm import tqdm
+import datasets
+
+import uvicorn
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+def load_corpus(corpus_path: str):
+    corpus = datasets.load_dataset(
+        'json', 
+        data_files=corpus_path,
+        split="train",
+        num_proc=4
+    )
+    return corpus
+
+def read_jsonl(file_path):
+    data = []
+    with open(file_path, "r") as f:
+        for line in f:
+            data.append(json.loads(line))
+    return data
+
+def load_docs(corpus, doc_idxs):
+    results = [corpus[int(idx)] for idx in doc_idxs]
+    return results
+
+def load_model(model_path: str, use_fp16: bool = False):
+    model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+    model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
+    model.eval()
+    model.cuda()
+    if use_fp16: 
+        model = model.half()
+    tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True)
+    return model, tokenizer
+
+def pooling(
+    pooler_output,
+    last_hidden_state,
+    attention_mask = None,
+    pooling_method = "mean"
+):
+    if pooling_method == "mean":
+        last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
+        return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
+    elif pooling_method == "cls":
+        return last_hidden_state[:, 0]
+    elif pooling_method == "pooler":
+        return pooler_output
+    else:
+        raise NotImplementedError("Pooling method not implemented!")
+
+class Encoder:
+    def __init__(self, model_name, model_path, pooling_method, max_length, use_fp16):
+        self.model_name = model_name
+        self.model_path = model_path
+        self.pooling_method = pooling_method
+        self.max_length = max_length
+        self.use_fp16 = use_fp16
+
+        self.model, self.tokenizer = load_model(model_path=model_path, use_fp16=use_fp16)
+        self.model.eval()
+
+    @torch.no_grad()
+    def encode(self, query_list: List[str], is_query=True) -> np.ndarray:
+        # processing query for different encoders
+        if isinstance(query_list, str):
+            query_list = [query_list]
+
+        if "e5" in self.model_name.lower():
+            if is_query:
+                query_list = [f"query: {query}" for query in query_list]
+            else:
+                query_list = [f"passage: {query}" for query in query_list]
+
+        if "bge" in self.model_name.lower():
+            if is_query:
+                query_list = [f"Represent this sentence for searching relevant passages: {query}" for query in query_list]
+
+        inputs = self.tokenizer(query_list,
+                                max_length=self.max_length,
+                                padding=True,
+                                truncation=True,
+                                return_tensors="pt"
+                                )
+        inputs = {k: v.cuda() for k, v in inputs.items()}
+
+        if "T5" in type(self.model).__name__:
+            # T5-based retrieval model
+            decoder_input_ids = torch.zeros(
+                (inputs['input_ids'].shape[0], 1), dtype=torch.long
+            ).to(inputs['input_ids'].device)
+            output = self.model(
+                **inputs, decoder_input_ids=decoder_input_ids, return_dict=True
+            )
+            query_emb = output.last_hidden_state[:, 0, :]
+        else:
+            output = self.model(**inputs, return_dict=True)
+            query_emb = pooling(output.pooler_output,
+                                output.last_hidden_state,
+                                inputs['attention_mask'],
+                                self.pooling_method)
+            if "dpr" not in self.model_name.lower():
+                query_emb = torch.nn.functional.normalize(query_emb, dim=-1)
+
+        query_emb = query_emb.detach().cpu().numpy()
+        query_emb = query_emb.astype(np.float32, order="C")
+        
+        del inputs, output
+        torch.cuda.empty_cache()
+
+        return query_emb
+
+class BaseRetriever:
+    def __init__(self, config):
+        self.config = config
+        self.retrieval_method = config.retrieval_method
+        self.topk = config.retrieval_topk
+        
+        self.index_path = config.index_path
+        self.corpus_path = config.corpus_path
+
+    def _search(self, query: str, num: int, return_score: bool):
+        raise NotImplementedError
+
+    def _batch_search(self, query_list: List[str], num: int, return_score: bool):
+        raise NotImplementedError
+
+    def search(self, query: str, num: int = None, return_score: bool = False):
+        return self._search(query, num, return_score)
+    
+    def batch_search(self, query_list: List[str], num: int = None, return_score: bool = False):
+        return self._batch_search(query_list, num, return_score)
+
+class BM25Retriever(BaseRetriever):
+    def __init__(self, config):
+        super().__init__(config)
+        from pyserini.search.lucene import LuceneSearcher
+        self.searcher = LuceneSearcher(self.index_path)
+        self.contain_doc = self._check_contain_doc()
+        if not self.contain_doc:
+            self.corpus = load_corpus(self.corpus_path)
+        self.max_process_num = 8
+    
+    def _check_contain_doc(self):
+        return self.searcher.doc(0).raw() is not None
+
+    def _search(self, query: str, num: int = None, return_score: bool = False):
+        if num is None:
+            num = self.topk
+        hits = self.searcher.search(query, num)
+        if len(hits) < 1:
+            if return_score:
+                return [], []
+            else:
+                return []
+        scores = [hit.score for hit in hits]
+        if len(hits) < num:
+            warnings.warn('Not enough documents retrieved!')
+        else:
+            hits = hits[:num]
+
+        if self.contain_doc:
+            all_contents = [
+                json.loads(self.searcher.doc(hit.docid).raw())['contents'] 
+                for hit in hits
+            ]
+            results = [
+                {
+                    'title': content.split("\n")[0].strip("\""),
+                    'text': "\n".join(content.split("\n")[1:]),
+                    'contents': content
+                } 
+                for content in all_contents
+            ]
+        else:
+            results = load_docs(self.corpus, [hit.docid for hit in hits])
+
+        if return_score:
+            return results, scores
+        else:
+            return results
+
+    def _batch_search(self, query_list: List[str], num: int = None, return_score: bool = False):
+        results = []
+        scores = []
+        for query in query_list:
+            item_result, item_score = self._search(query, num, True)
+            results.append(item_result)
+            scores.append(item_score)
+        if return_score:
+            return results, scores
+        else:
+            return results
+
+class DenseRetriever(BaseRetriever):
+    def __init__(self, config):
+        super().__init__(config)
+        self.index = faiss.read_index(self.index_path)
+        if config.faiss_gpu:
+            co = faiss.GpuMultipleClonerOptions()
+            co.useFloat16 = True
+            co.shard = True
+            self.index = faiss.index_cpu_to_all_gpus(self.index, co=co)
+
+        self.corpus = load_corpus(self.corpus_path)
+        self.encoder = Encoder(
+            model_name = self.retrieval_method,
+            model_path = config.retrieval_model_path,
+            pooling_method = config.retrieval_pooling_method,
+            max_length = config.retrieval_query_max_length,
+            use_fp16 = config.retrieval_use_fp16
+        )
+        self.topk = config.retrieval_topk
+        self.batch_size = config.retrieval_batch_size
+
+    def _search(self, query: str, num: int = None, return_score: bool = False):
+        if num is None:
+            num = self.topk
+        query_emb = self.encoder.encode(query)
+        scores, idxs = self.index.search(query_emb, k=num)
+        idxs = idxs[0]
+        scores = scores[0]
+        results = load_docs(self.corpus, idxs)
+        if return_score:
+            return results, scores.tolist()
+        else:
+            return results
+
+    def _batch_search(self, query_list: List[str], num: int = None, return_score: bool = False):
+        if isinstance(query_list, str):
+            query_list = [query_list]
+        if num is None:
+            num = self.topk
+        
+        results = []
+        scores = []
+        for start_idx in tqdm(range(0, len(query_list), self.batch_size), desc='Retrieval process: '):
+            query_batch = query_list[start_idx:start_idx + self.batch_size]
+            batch_emb = self.encoder.encode(query_batch)
+            batch_scores, batch_idxs = self.index.search(batch_emb, k=num)
+            batch_scores = batch_scores.tolist()
+            batch_idxs = batch_idxs.tolist()
+
+            # load_docs is not vectorized, but is a python list approach
+            flat_idxs = sum(batch_idxs, [])
+            batch_results = load_docs(self.corpus, flat_idxs)
+            # chunk them back
+            batch_results = [batch_results[i*num : (i+1)*num] for i in range(len(batch_idxs))]
+            
+            results.extend(batch_results)
+            scores.extend(batch_scores)
+            
+            del batch_emb, batch_scores, batch_idxs, query_batch, flat_idxs, batch_results
+            torch.cuda.empty_cache()
+            
+        if return_score:
+            return results, scores
+        else:
+            return results
+
+def get_retriever(config):
+    if config.retrieval_method == "bm25":
+        return BM25Retriever(config)
+    else:
+        return DenseRetriever(config)
+
+
+#####################################
+# FastAPI server below
+#####################################
+
+class Config:
+    """
+    Minimal config class (simulating your argparse) 
+    Replace this with your real arguments or load them dynamically.
+    """
+    def __init__(
+        self, 
+        retrieval_method: str = "bm25", 
+        retrieval_topk: int = 10,
+        index_path: str = "./index/bm25",
+        corpus_path: str = "./data/corpus.jsonl",
+        dataset_path: str = "./data",
+        data_split: str = "train",
+        faiss_gpu: bool = True,
+        retrieval_model_path: str = "./model",
+        retrieval_pooling_method: str = "mean",
+        retrieval_query_max_length: int = 256,
+        retrieval_use_fp16: bool = False,
+        retrieval_batch_size: int = 128
+    ):
+        self.retrieval_method = retrieval_method
+        self.retrieval_topk = retrieval_topk
+        self.index_path = index_path
+        self.corpus_path = corpus_path
+        self.dataset_path = dataset_path
+        self.data_split = data_split
+        self.faiss_gpu = faiss_gpu
+        self.retrieval_model_path = retrieval_model_path
+        self.retrieval_pooling_method = retrieval_pooling_method
+        self.retrieval_query_max_length = retrieval_query_max_length
+        self.retrieval_use_fp16 = retrieval_use_fp16
+        self.retrieval_batch_size = retrieval_batch_size
+
+
+class QueryRequest(BaseModel):
+    queries: List[str]
+    topk: Optional[int] = None
+    return_scores: bool = False
+
+
+app = FastAPI()
+
+@app.post("/retrieve")
+def retrieve_endpoint(request: QueryRequest):
+    """
+    Endpoint that accepts queries and performs retrieval.
+    Input format:
+    {
+      "queries": ["What is Python?", "Tell me about neural networks."],
+      "topk": 3,
+      "return_scores": true
+    }
+    """
+    if not request.topk:
+        request.topk = config.retrieval_topk  # fallback to default
+
+    # Perform batch retrieval
+    results, scores = retriever.batch_search(
+        query_list=request.queries,
+        num=request.topk,
+        return_score=request.return_scores
+    )
+    
+    # Format response
+    resp = []
+    for i, single_result in enumerate(results):
+        if request.return_scores:
+            # If scores are returned, combine them with results
+            combined = []
+            for doc, score in zip(single_result, scores[i]):
+                combined.append({"document": doc, "score": score})
+            resp.append(combined)
+        else:
+            resp.append(single_result)
+    return {"result": resp}
+
+
+if __name__ == "__main__":
+    
+    parser = argparse.ArgumentParser(description="Launch the local faiss retriever.")
+    parser.add_argument("--index_path", type=str, default="/home/peterjin/mnt/index/wiki-18/e5_Flat.index", help="Corpus indexing file.")
+    parser.add_argument("--corpus_path", type=str, default="/home/peterjin/mnt/data/retrieval-corpus/wiki-18.jsonl", help="Local corpus file.")
+    parser.add_argument("--topk", type=int, default=3, help="Number of retrieved passages for one query.")
+    parser.add_argument("--retriever_name", type=str, default="e5", help="Name of the retriever model.")
+    parser.add_argument("--retriever_model", type=str, default="intfloat/e5-base-v2", help="Path of the retriever model.")
+    parser.add_argument('--faiss_gpu', action='store_true', help='Use GPU for computation')
+
+    args = parser.parse_args()
+    
+    # 1) Build a config (could also parse from arguments).
+    #    In real usage, you'd parse your CLI arguments or environment variables.
+    config = Config(
+        retrieval_method = args.retriever_name,  # or "dense"
+        index_path=args.index_path,
+        corpus_path=args.corpus_path,
+        retrieval_topk=args.topk,
+        faiss_gpu=args.faiss_gpu,
+        retrieval_model_path=args.retriever_model,
+        retrieval_pooling_method="mean",
+        retrieval_query_max_length=256,
+        retrieval_use_fp16=True,
+        retrieval_batch_size=512,
+    )
+
+    # 2) Instantiate a global retriever so it is loaded once and reused.
+    retriever = get_retriever(config)
+    
+    # 3) Launch the server. By default, it listens on http://127.0.0.1:8000
+    uvicorn.run(app, host="0.0.0.0", port=8000)

code/RL_model/verl/Search-R1/search_r1/search/serp_search_server.py

@@ -0,0 +1,112 @@
+import os
+import requests
+from fastapi import FastAPI
+from pydantic import BaseModel
+from typing import List, Optional, Dict
+from concurrent.futures import ThreadPoolExecutor
+import argparse
+import uvicorn
+
+parser = argparse.ArgumentParser(description="Launch online search server.")
+parser.add_argument('--search_url', type=str, required=True, 
+                    help="URL for search engine (e.g. https://serpapi.com/search)")
+parser.add_argument('--topk', type=int, default=3, 
+                    help="Number of results to return per query")
+parser.add_argument('--serp_api_key', type=str, default=None, 
+                    help="SerpAPI key for online search")
+parser.add_argument('--serp_engine', type=str, default="google", 
+                    help="SerpAPI engine for online search")
+args = parser.parse_args()
+
+# --- Config ---
+class OnlineSearchConfig:
+    def __init__(
+        self,
+        search_url: str = "https://serpapi.com/search",
+        topk: int = 3,
+        serp_api_key: Optional[str] = None,
+        serp_engine: Optional[str] = None,
+    ):
+        self.search_url = search_url
+        self.topk = topk
+        self.serp_api_key = serp_api_key
+        self.serp_engine = serp_engine
+
+
+# --- Online Search Wrapper ---
+class OnlineSearchEngine:
+    def __init__(self, config: OnlineSearchConfig):
+        self.config = config
+
+    def _search_query(self, query: str):
+        params = {
+            "engine": self.config.serp_engine,
+            "q": query,
+            "api_key": self.config.serp_api_key,
+        }
+        response = requests.get(self.config.search_url, params=params)
+        return response.json()
+
+    def batch_search(self, queries: List[str]):
+        results = []
+        with ThreadPoolExecutor() as executor:
+            for result in executor.map(self._search_query, queries):
+                results.append(self._process_result(result))
+        return results
+
+    def _process_result(self, search_result: Dict):
+        results = []
+        
+        answer_box = search_result.get('answer_box', {})
+        if answer_box:
+            title = answer_box.get('title', 'No title.')
+            snippet = answer_box.get('snippet', 'No snippet available.')
+            results.append({
+                'document': {"contents": f'\"{title}\"\n{snippet}'},
+            })
+
+        organic_results = search_result.get('organic_results', [])
+        for _, result in enumerate(organic_results[:self.config.topk]):
+            title = result.get('title', 'No title.')
+            snippet = result.get('snippet', 'No snippet available.')
+            results.append({
+                'document': {"contents": f'\"{title}\"\n{snippet}'},
+            })
+
+        related_results = search_result.get('related_questions', [])
+        for _, result in enumerate(related_results[:self.config.topk]):
+            title = result.get('question', 'No title.')  # question is the title here
+            snippet = result.get('snippet', 'No snippet available.')
+            results.append({
+                'document': {"contents": f'\"{title}\"\n{snippet}'},
+            })
+
+        return results
+
+
+# --- FastAPI Setup ---
+app = FastAPI(title="Online Search Proxy Server")
+
+class SearchRequest(BaseModel):
+    queries: List[str]
+
+# Instantiate global config + engine
+config = OnlineSearchConfig(
+    search_url=args.search_url,
+    topk=args.topk,
+    serp_api_key=args.serp_api_key,
+    serp_engine=args.serp_engine,
+)
+engine = OnlineSearchEngine(config)
+
+# --- Routes ---
+@app.post("/retrieve")
+def search_endpoint(request: SearchRequest):
+    results = engine.batch_search(request.queries)
+    return {"result": results}
+
+## return {"result": List[List[{'document': {"id": xx, "content": "title" + \n + "content"}, 'score': xx}]]}
+
+if __name__ == "__main__":
+    # 3) Launch the server. By default, it listens on http://127.0.0.1:8000
+    uvicorn.run(app, host="0.0.0.0", port=8000)

code/RL_model/verl/Search-R1/verl.egg-info/SOURCES.txt

@@ -0,0 +1,190 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+./search_r1/__init__.py
+./search_r1/llm_agent/__init__.py
+./search_r1/llm_agent/generation.py
+./search_r1/llm_agent/tensor_helper.py
+./verl/__init__.py
+./verl/protocol.py
+./verl/models/__init__.py
+./verl/models/registry.py
+./verl/models/weight_loader_registry.py
+./verl/models/llama/__init__.py
+./verl/models/llama/megatron/__init__.py
+./verl/models/llama/megatron/modeling_llama_megatron.py
+./verl/models/llama/megatron/checkpoint_utils/__init__.py
+./verl/models/llama/megatron/checkpoint_utils/llama_loader.py
+./verl/models/llama/megatron/checkpoint_utils/llama_saver.py
+./verl/models/llama/megatron/layers/__init__.py
+./verl/models/llama/megatron/layers/parallel_attention.py
+./verl/models/llama/megatron/layers/parallel_decoder.py
+./verl/models/llama/megatron/layers/parallel_linear.py
+./verl/models/llama/megatron/layers/parallel_mlp.py
+./verl/models/llama/megatron/layers/parallel_rmsnorm.py
+./verl/models/transformers/__init__.py
+./verl/models/transformers/llama.py
+./verl/models/transformers/monkey_patch.py
+./verl/models/transformers/qwen2.py
+./verl/single_controller/__init__.py
+./verl/single_controller/base/__init__.py
+./verl/single_controller/base/decorator.py
+./verl/single_controller/base/worker.py
+./verl/single_controller/base/worker_group.py
+./verl/single_controller/base/megatron/__init__.py
+./verl/single_controller/base/megatron/worker.py
+./verl/single_controller/base/megatron/worker_group.py
+./verl/single_controller/base/register_center/__init__.py
+./verl/single_controller/base/register_center/ray.py
+./verl/single_controller/ray/__init__.py
+./verl/single_controller/ray/base.py
+./verl/single_controller/ray/megatron.py
+./verl/third_party/__init__.py
+./verl/third_party/vllm/__init__.py
+./verl/third_party/vllm/vllm_v_0_3_1/__init__.py
+./verl/third_party/vllm/vllm_v_0_3_1/arg_utils.py
+./verl/third_party/vllm/vllm_v_0_3_1/config.py
+./verl/third_party/vllm/vllm_v_0_3_1/llm.py
+./verl/third_party/vllm/vllm_v_0_3_1/llm_engine_sp.py
+./verl/third_party/vllm/vllm_v_0_3_1/model_loader.py
+./verl/third_party/vllm/vllm_v_0_3_1/model_runner.py
+./verl/third_party/vllm/vllm_v_0_3_1/parallel_state.py
+./verl/third_party/vllm/vllm_v_0_3_1/tokenizer.py
+./verl/third_party/vllm/vllm_v_0_3_1/weight_loaders.py
+./verl/third_party/vllm/vllm_v_0_3_1/worker.py
+./verl/third_party/vllm/vllm_v_0_4_2/__init__.py
+./verl/third_party/vllm/vllm_v_0_4_2/arg_utils.py
+./verl/third_party/vllm/vllm_v_0_4_2/config.py
+./verl/third_party/vllm/vllm_v_0_4_2/dtensor_weight_loaders.py
+./verl/third_party/vllm/vllm_v_0_4_2/hf_weight_loader.py
+./verl/third_party/vllm/vllm_v_0_4_2/llm.py
+./verl/third_party/vllm/vllm_v_0_4_2/llm_engine_sp.py
+./verl/third_party/vllm/vllm_v_0_4_2/megatron_weight_loaders.py
+./verl/third_party/vllm/vllm_v_0_4_2/model_loader.py
+./verl/third_party/vllm/vllm_v_0_4_2/model_runner.py
+./verl/third_party/vllm/vllm_v_0_4_2/parallel_state.py
+./verl/third_party/vllm/vllm_v_0_4_2/spmd_gpu_executor.py
+./verl/third_party/vllm/vllm_v_0_4_2/tokenizer.py
+./verl/third_party/vllm/vllm_v_0_4_2/worker.py
+./verl/third_party/vllm/vllm_v_0_5_4/__init__.py
+./verl/third_party/vllm/vllm_v_0_5_4/arg_utils.py
+./verl/third_party/vllm/vllm_v_0_5_4/config.py
+./verl/third_party/vllm/vllm_v_0_5_4/dtensor_weight_loaders.py
+./verl/third_party/vllm/vllm_v_0_5_4/hf_weight_loader.py
+./verl/third_party/vllm/vllm_v_0_5_4/llm.py
+./verl/third_party/vllm/vllm_v_0_5_4/llm_engine_sp.py
+./verl/third_party/vllm/vllm_v_0_5_4/megatron_weight_loaders.py
+./verl/third_party/vllm/vllm_v_0_5_4/model_loader.py
+./verl/third_party/vllm/vllm_v_0_5_4/model_runner.py
+./verl/third_party/vllm/vllm_v_0_5_4/parallel_state.py
+./verl/third_party/vllm/vllm_v_0_5_4/spmd_gpu_executor.py
+./verl/third_party/vllm/vllm_v_0_5_4/tokenizer.py
+./verl/third_party/vllm/vllm_v_0_5_4/worker.py
+./verl/third_party/vllm/vllm_v_0_6_3/__init__.py
+./verl/third_party/vllm/vllm_v_0_6_3/arg_utils.py
+./verl/third_party/vllm/vllm_v_0_6_3/config.py
+./verl/third_party/vllm/vllm_v_0_6_3/dtensor_weight_loaders.py
+./verl/third_party/vllm/vllm_v_0_6_3/hf_weight_loader.py
+./verl/third_party/vllm/vllm_v_0_6_3/llm.py
+./verl/third_party/vllm/vllm_v_0_6_3/llm_engine_sp.py
+./verl/third_party/vllm/vllm_v_0_6_3/megatron_weight_loaders.py
+./verl/third_party/vllm/vllm_v_0_6_3/model_loader.py
+./verl/third_party/vllm/vllm_v_0_6_3/model_runner.py
+./verl/third_party/vllm/vllm_v_0_6_3/parallel_state.py
+./verl/third_party/vllm/vllm_v_0_6_3/spmd_gpu_executor.py
+./verl/third_party/vllm/vllm_v_0_6_3/tokenizer.py
+./verl/third_party/vllm/vllm_v_0_6_3/worker.py
+./verl/trainer/__init__.py
+./verl/trainer/fsdp_sft_trainer.py
+./verl/trainer/main_eval.py
+./verl/trainer/main_generation.py
+./verl/trainer/main_ppo.py
+./verl/trainer/main_ppo_format.py
+./verl/trainer/config/evaluation.yaml
+./verl/trainer/config/generation.yaml
+./verl/trainer/config/ppo_megatron_trainer.yaml
+./verl/trainer/config/ppo_trainer.yaml
+./verl/trainer/config/sft_trainer.yaml
+./verl/trainer/ppo/__init__.py
+./verl/trainer/ppo/core_algos.py
+./verl/trainer/ppo/ray_trainer.py
+./verl/utils/__init__.py
+./verl/utils/config.py
+./verl/utils/distributed.py
+./verl/utils/flops_counter.py
+./verl/utils/fs.py
+./verl/utils/fsdp_utils.py
+./verl/utils/hdfs_io.py
+./verl/utils/import_utils.py
+./verl/utils/logging_utils.py
+./verl/utils/megatron_utils.py
+./verl/utils/memory_buffer.py
+./verl/utils/model.py
+./verl/utils/py_functional.py
+./verl/utils/ray_utils.py
+./verl/utils/seqlen_balancing.py
+./verl/utils/tokenizer.py
+./verl/utils/torch_dtypes.py
+./verl/utils/torch_functional.py
+./verl/utils/tracking.py
+./verl/utils/ulysses.py
+./verl/utils/dataset/__init__.py
+./verl/utils/dataset/rl_dataset.py
+./verl/utils/dataset/rm_dataset.py
+./verl/utils/debug/__init__.py
+./verl/utils/debug/performance.py
+./verl/utils/debug/trajectory_tracker.py
+./verl/utils/logger/__init__.py
+./verl/utils/logger/aggregate_logger.py
+./verl/utils/megatron/__init__.py
+./verl/utils/megatron/memory.py
+./verl/utils/megatron/optimizer.py
+./verl/utils/megatron/optimizer_config.py
+./verl/utils/megatron/pipeline_parallel.py
+./verl/utils/megatron/sequence_parallel.py
+./verl/utils/megatron/tensor_parallel.py
+./verl/utils/rendezvous/__init__.py
+./verl/utils/rendezvous/ray_backend.py
+./verl/utils/reward_score/__init__.py
+./verl/utils/reward_score/countdown.py
+./verl/utils/reward_score/gsm8k.py
+./verl/utils/reward_score/math.py
+./verl/utils/reward_score/multiply.py
+./verl/utils/reward_score/qa_em.py
+./verl/utils/reward_score/qa_em_format.py
+./verl/version/version
+./verl/workers/__init__.py
+./verl/workers/fsdp_workers.py
+./verl/workers/megatron_workers.py
+./verl/workers/actor/__init__.py
+./verl/workers/actor/base.py
+./verl/workers/actor/dp_actor.py
+./verl/workers/actor/megatron_actor.py
+./verl/workers/critic/__init__.py
+./verl/workers/critic/base.py
+./verl/workers/critic/dp_critic.py
+./verl/workers/critic/megatron_critic.py
+./verl/workers/reward_model/__init__.py
+./verl/workers/reward_model/base.py
+./verl/workers/reward_model/megatron/__init__.py
+./verl/workers/reward_model/megatron/reward_model.py
+./verl/workers/rollout/__init__.py
+./verl/workers/rollout/base.py
+./verl/workers/rollout/hf_rollout.py
+./verl/workers/rollout/tokenizer.py
+./verl/workers/rollout/naive/__init__.py
+./verl/workers/rollout/naive/naive_rollout.py
+./verl/workers/rollout/vllm_rollout/__init__.py
+./verl/workers/rollout/vllm_rollout/vllm_rollout.py
+./verl/workers/sharding_manager/__init__.py
+./verl/workers/sharding_manager/base.py
+./verl/workers/sharding_manager/fsdp_ulysses.py
+./verl/workers/sharding_manager/fsdp_vllm.py
+./verl/workers/sharding_manager/megatron_vllm.py
+verl.egg-info/PKG-INFO
+verl.egg-info/SOURCES.txt
+verl.egg-info/dependency_links.txt
+verl.egg-info/requires.txt
+verl.egg-info/top_level.txt
+verl/version/version

code/RL_model/verl/Search-R1/verl/single_controller/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__)))
+
+with open(os.path.join(version_folder, 'version/version')) as f:
+    __version__ = f.read().strip()

code/RL_model/verl/Search-R1/verl/trainer/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

code/RL_model/verl/Search-R1/verl/trainer/main_eval.py

@@ -0,0 +1,69 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Offline evaluate the performance of a generated file using reward model and ground truth verifier.
+The input is a parquet file that contains N generated sequences and (optional) the ground truth.
+
+"""
+
+import hydra
+from verl.utils.fs import copy_local_path_from_hdfs
+from verl.utils.reward_score import math, gsm8k
+import pandas as pd
+import numpy as np
+
+
+def select_reward_fn(data_source):
+    if data_source == 'lighteval/MATH':
+        return math.compute_score
+    else:
+        raise NotImplementedError
+
+
+@hydra.main(config_path='config', config_name='evaluation', version_base=None)
+def main(config):
+    local_path = copy_local_path_from_hdfs(config.data.path)
+    dataset = pd.read_parquet(local_path)
+    prompts = dataset[config.data.prompt_key]
+    responses = dataset[config.data.response_key]
+    data_sources = dataset[config.data.data_source_key]
+    reward_model_data = dataset[config.data.reward_model_key]
+
+    passes = 0
+
+    total = len(dataset)
+
+    for i in range(total):
+        response_lst = responses[i]
+        data_source = data_sources[i]
+        # select reward score based on data_source
+        prompt = prompts[i]
+        reward_data = reward_model_data[i]
+        reward_fn = select_reward_fn(data_source)
+        ground_truth = reward_data['ground_truth']
+        score_lst = []
+        for r in response_lst:
+            score = reward_fn(r, ground_truth)
+            score_lst.append(score)
+
+        max_score = np.max(score_lst)
+
+        if max_score == 1:
+            passes += 1
+
+    print(f'pass@5: {passes / total}')
+
+
+if __name__ == '__main__':
+    main()

code/RL_model/verl/Search-R1/verl/utils/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import tokenizer
+from .tokenizer import *
+
+__all__ = tokenizer.__all__

code/RL_model/verl/Search-R1/verl/utils/config.py

@@ -0,0 +1,23 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict
+
+from omegaconf import DictConfig
+
+
+def update_dict_with_config(dictionary: Dict, config: DictConfig):
+    for key in dictionary:
+        if hasattr(config, key):
+            dictionary[key] = getattr(config, key)