| project: "SCIENCEWORLD_TCOD" |
| name: "scienceworld_tcod_f2b" |
| checkpoint_root_dir: ./checkpoints |
| continue_from_checkpoint: false |
|
|
| algorithm: |
| sample_strategy: staleness_control |
| sample_strategy_args: |
| max_staleness: 2 |
| algorithm_type: on_policy_distill |
| advantage_fn: multi_turn_opd |
| repeat_times: 1 |
| advantage_fn_args: |
| kl_coef: 1.0 |
| optimizer: |
| lr: 1e-6 |
|
|
| model: |
| |
| model_path: Qwen/Qwen2.5-1.5B-Instruct |
| max_prompt_tokens: 10240 |
| max_response_tokens: 512 |
|
|
| cluster: |
| node_num: 1 |
| gpu_per_node: 8 |
|
|
| buffer: |
| total_steps: 250 |
| batch_size: 16 |
| train_batch_size: 64 |
| explorer_input: |
| taskset: |
| name: sciworld |
| storage_type: file |
| path: /path/to/sciworld_data |
| split: train |
| format: |
| prompt_key: 'task_desc' |
| rollout_args: |
| temperature: 1.0 |
| logprobs: 0 |
| workflow_args: |
| temperature: 1.0 |
| max_env_steps: 30 |
| total_steps: 250 |
| checkpoint_strategy: linear |
| checkpoint_steps: 6 |
| eval_tasksets: |
| - name: sciworld_eval |
| storage_type: file |
| path: /path/to/sciworld_data |
| split: test |
| total_steps: 10 |
| task_selector: |
| selector_type: random |
| seed: 42 |
| format: |
| prompt_key: 'task_desc' |
| rollout_args: |
| temperature: 0.4 |
| logprobs: 0 |
| workflow_args: |
| temperature: 1.0 |
| max_env_steps: 30 |
| total_steps: 250 |
| checkpoint_strategy: linear |
| checkpoint_steps: 6 |
| default_workflow_type: 'TCOD_f2b_scienceworld_workflow' |
| trainer_input: |
| experience_buffer: |
| name: scienceworld_tcod_f2b_buffer |
| storage_type: queue |
| path: 'sqlite:///scienceworld_tcod_f2b_buffer.db' |
|
|
| explorer: |
| eval_interval: 5 |
| runner_per_model: 8 |
| max_timeout: 3600 |
| rollout_model: |
| engine_num: 2 |
| tensor_parallel_size: 2 |
| enable_prefix_caching: false |
| enforce_eager: true |
| dtype: bfloat16 |
| seed: 42 |
| gpu_memory_utilization: 0.7 |
| enable_chunked_prefill: true |
| auxiliary_models: |
| |
| - model_path: /path/to/teacher |
| engine_num: 1 |
| tensor_parallel_size: 2 |
| enable_prefix_caching: false |
| enforce_eager: true |
| dtype: bfloat16 |
| seed: 42 |
| max_model_len: 20480 |
| max_prompt_tokens: 16384 |
| max_response_tokens: 512 |
| env_vars: |
| TMPDIR: /dev/shm/tmp |
| RAY_TMPDIR: /dev/shm/ray_tmp |
|
|
| synchronizer: |
| sync_method: 'nccl' |
| sync_style: 'dynamic_by_explorer' |
| sync_interval: 1 |
| sync_timeout: 3600 |
|
|
| trainer: |
| total_steps: 250 |
| save_interval: 250 |
| grad_clip: 1.0 |
| use_dynamic_bsz: true |
| max_token_len_per_gpu: 16384 |
| ulysses_sequence_parallel_size: 2 |
|
|
| monitor: |
| monitor_type: wandb |
|
|
