project: "SCIENCEWORLD_TCOD"
name: "scienceworld_tcod_f2b"
checkpoint_root_dir: ./checkpoints
continue_from_checkpoint: false

algorithm:
  sample_strategy: staleness_control
  sample_strategy_args:
    max_staleness: 2
  algorithm_type: on_policy_distill
  advantage_fn: multi_turn_opd
  repeat_times: 1
  advantage_fn_args:
    kl_coef: 1.0
  optimizer:
    lr: 1e-6

model:
  # Student model
  model_path: Qwen/Qwen2.5-1.5B-Instruct
  max_prompt_tokens: 10240
  max_response_tokens: 512

cluster:
  node_num: 1
  gpu_per_node: 8

buffer:
  total_steps: 250
  batch_size: 16
  train_batch_size: 64
  explorer_input:
    taskset:
      name: sciworld
      storage_type: file
      path: /path/to/sciworld_data
      split: train
      format:
        prompt_key: 'task_desc'
      rollout_args:
        temperature: 1.0
        logprobs: 0
      workflow_args:
        temperature: 1.0
        max_env_steps: 30
        total_steps: 250
        checkpoint_strategy: linear
        checkpoint_steps: 6
    eval_tasksets:
      - name: sciworld_eval
        storage_type: file
        path: /path/to/sciworld_data
        split: test
        total_steps: 10
        task_selector:
          selector_type: random
          seed: 42
        format:
          prompt_key: 'task_desc'
        rollout_args:
          temperature: 0.4
          logprobs: 0
        workflow_args:
          temperature: 1.0
          max_env_steps: 30
          total_steps: 250
          checkpoint_strategy: linear
          checkpoint_steps: 6
    default_workflow_type: 'TCOD_f2b_scienceworld_workflow'
  trainer_input:
    experience_buffer:
      name: scienceworld_tcod_f2b_buffer
      storage_type: queue
      path: 'sqlite:///scienceworld_tcod_f2b_buffer.db'

explorer:
  eval_interval: 5
  runner_per_model: 8
  max_timeout: 3600
  rollout_model:
    engine_num: 2
    tensor_parallel_size: 2
    enable_prefix_caching: false
    enforce_eager: true
    dtype: bfloat16
    seed: 42
    gpu_memory_utilization: 0.7
    enable_chunked_prefill: true
  auxiliary_models:
    # Teacher model
    - model_path: /path/to/teacher
      engine_num: 1
      tensor_parallel_size: 2
      enable_prefix_caching: false
      enforce_eager: true
      dtype: bfloat16
      seed: 42
      max_model_len: 20480
      max_prompt_tokens: 16384
      max_response_tokens: 512
  env_vars:
    TMPDIR: /dev/shm/tmp
    RAY_TMPDIR: /dev/shm/ray_tmp

synchronizer:
  sync_method: 'nccl'
  sync_style: 'dynamic_by_explorer'
  sync_interval: 1
  sync_timeout: 3600

trainer:
  total_steps: 250
  save_interval: 250
  grad_clip: 1.0
  use_dynamic_bsz: true
  max_token_len_per_gpu: 16384
  ulysses_sequence_parallel_size: 2

monitor:
  monitor_type: wandb