Upload MatchTIR-4B-extra-thinkback-toolfix1 global_step_16

MatchTIR-4B-extra-thinkback-toolfix1/resolved_config.yaml

@@ -0,0 +1,316 @@
+data:
+  tokenizer: null
+  train_files: Data/golden_train_235b_deepseek_merged_clean_plus_32b_extra.segmented_clean.parquet
+  val_files: Data/test.parquet
+  prompt_key: messages
+  reward_fn_key: data_source
+  tools_key: tools
+  max_prompt_length: 7000
+  max_response_length: 23000
+  train_batch_size: 256
+  val_batch_size: 256
+  return_raw_input_ids: false
+  return_raw_chat: false
+  return_full_prompt: false
+  shuffle: true
+  filter_overlong_prompts: true
+  filter_overlong_prompts_workers: 1
+  truncation: error
+  image_key: images
+  video_key: videos
+  custom_cls:
+    path: null
+    name: null
+  enable_thinking: true
+  sft_target_key: raw_output
+  sft_mode: segmented_prefix
+  segmented_prefix_files: Data/segmentation/54_hybrid.cleaned.plus_32b_extra.segmented.jsonl
+  segmented_prefix_supervise_visible_prefix: true
+  segmented_prefix_supervise_tool_call: true
+  random_pure_rl_prompt_proportion: 0.0
+  random_pure_rl_prompt_schedule: fix
+  random_pure_rl_prompt_start_proportion: 0.0
+  random_pure_rl_prompt_final_proportion: 0.0
+  random_pure_rl_prompt_annealing_ratio: 1.0
+  random_pure_rl_prompt_seed: 0
+  random_pure_rl_prompt_resample_each_epoch: true
+  sft_turn_intra_annealing: false
+  system_style: Qwen3
+  adaptive_next_turn_supervision:
+    enabled: false
+    reward_turn_proxy_field: turn_proxy_json
+    segmented_turns_field: segmented_turns_json
+    plan_field: adaptive_supervision_plan_json
+    strong_sft_floor: 0.8
+    medium_sft_floor: 0.6
+    weak_sft_floor: 0.5
+    allow_max_segmented_level: L2
+    conservative_tool_patterns:
+    - initial_attempt
+    - compare_then_call
+    bad_quality_flags:
+    - fallback_segmentation
+    - needs_intra_paragraph_cut
+    - duplicate_call_preview_in_reasoning
+    soft_bad_quality_flags:
+    - assistant_role_token_noise
+    - long_redundant_reasoning
+  dataloader_num_workers: 0
+  sft_proportion: 1.0
+  sft_whole_response: false
+  sft_turn_mode: true
+  sft_turn_uniform_proportion: true
+  sft_annealing: true
+  sft_annealing_ratio: 1.0
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen3-4B
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    enable_activation_offload: false
+    use_remove_padding: true
+    use_liger: false
+    use_fused_kernels: false
+    trust_remote_code: false
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 32
+    ppo_micro_batch_size: null
+    ppo_micro_batch_size_per_gpu: null
+    use_dynamic_bsz: true
+    ppo_max_token_len_per_gpu: 15000
+    grad_clip: 1.0
+    clip_ratio: 0.2
+    clip_ratio_low: 0.2
+    clip_ratio_high: 0.2
+    clip_ratio_c: 3.0
+    loss_agg_mode: seq-mean-token-mean
+    entropy_coeff: 0.001
+    use_hint_for_log_prob: true
+    segmented_prefix_thinking_sft_loss_scale: 1.0
+    segmented_prefix_tool_call_sft_loss_scale: 1.0
+    segmented_prefix_tool_call_sft_loss_scale_annealing: false
+    segmented_prefix_tool_call_sft_loss_final_scale: 1.0
+    segmented_prefix_tool_call_sft_loss_annealing_ratio: 1.0
+    use_kl_loss: true
+    use_torch_compile: true
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 2
+    checkpoint:
+      contents:
+      - model
+      - optimizer
+      - extra
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps: -1
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: 0.0
+      num_cycles: 0.5
+      warmup_style: constant
+      total_training_steps: 24
+      weight_decay: 0.01
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: true
+      optimizer_offload: true
+      offload_policy: false
+      reshard_after_forward: true
+      fsdp_size: -1
+    use_sft_loss: true
+    sft_loss_coef: 0.1
+  ref:
+    strategy: fsdp
+    fsdp_config:
+      param_offload: true
+      reshard_after_forward: true
+      wrap_policy:
+        min_num_params: 0
+    use_torch_compile: true
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: null
+    log_prob_use_dynamic_bsz: true
+    log_prob_max_token_len_per_gpu: 15000
+    ulysses_sequence_parallel_size: 2
+  rollout:
+    name: vllm
+    mode: sync
+    chat_scheduler: null
+    temperature: 1.0
+    top_k: -1
+    top_p: 1
+    use_fire_sampling: false
+    prompt_length: 7000
+    response_length: 23000
+    dtype: bfloat16
+    gpu_memory_utilization: 0.7
+    ignore_eos: false
+    enforce_eager: true
+    free_cache_engine: true
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 1
+    max_num_batched_tokens: 32768
+    max_model_len: null
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: null
+    log_prob_use_dynamic_bsz: true
+    log_prob_max_token_len_per_gpu: 15000
+    disable_log_stats: true
+    enable_chunked_prefill: true
+    do_sample: true
+    'n': 8
+    engine_kwargs:
+      vllm:
+        swap_space: null
+      sglang:
+        attention_backend: null
+    val_kwargs:
+      top_k: -1
+      top_p: 1.0
+      temperature: 0
+      'n': 1
+      do_sample: false
+    multi_turn:
+      enable: false
+      max_turns: null
+      tool_config_path: null
+      format: chatml
+    enable_thinking: true
+    max_turns: 10
+critic:
+  rollout_n: 8
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: 24
+    weight_decay: 0.01
+  model:
+    path: ~/models/deepseek-llm-7b-chat
+    tokenizer_path: Qwen/Qwen3-4B
+    override_config: {}
+    external_lib: null
+    enable_gradient_checkpointing: true
+    enable_activation_offload: false
+    use_remove_padding: false
+    trust_remote_code: false
+    fsdp_config:
+      param_offload: false
+      optimizer_offload: false
+      offload_policy: false
+      reshard_after_forward: true
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+  ppo_mini_batch_size: 32
+  ppo_micro_batch_size: null
+  ppo_micro_batch_size_per_gpu: null
+  forward_micro_batch_size: null
+  forward_micro_batch_size_per_gpu: null
+  use_dynamic_bsz: true
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: 32768
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: 1
+  shuffle: false
+  grad_clip: 'inf'
+  cliprange_value: 0.5
+  loss_agg_mode: seq-mean-token-mean
+  checkpoint:
+    contents:
+    - model
+    - optimizer
+    - extra
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: Qwen/Qwen3-4B
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: null
+    use_remove_padding: false
+    use_fused_kernels: false
+    trust_remote_code: false
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: false
+      reshard_after_forward: true
+      fsdp_size: -1
+  micro_batch_size: null
+  micro_batch_size_per_gpu: null
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: true
+  forward_max_token_len_per_gpu: 32768
+  reward_manager: tool
+  launch_reward_fn_async: false
+  sandbox_fusion:
+    url: null
+    max_concurrent: 64
+custom_reward_function:
+  path: Code/verl/utils/reward_score/tool.py
+  name: compute_process_KM
+  reward_kwargs:
+    emit_turn_proxy_diagnostics: true
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: grpo
+  norm_adv_by_std_in_grpo: true
+  use_kl_in_reward: false
+  dynamic_advantage:
+    enabled: false
+    schedule: linear
+    global_start: 0.8
+    global_end: 0.2
+    local_start: 0.2
+    local_end: 0.8
+    sign_conflict_enabled: true
+    sign_conflict_eps: 0.05
+    sign_conflict_local_weight: 0.8
+    clamp_progress: true
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+    horizon: 10000
+    target_kl: 0.1
+trainer:
+  balance_batch: true
+  total_epochs: 3
+  total_training_steps: null
+  project_name: MatchTIR
+  experiment_name: MatchTIR-4B-extra-thinkback-toolfix1
+  logger:
+  - console
+  - wandb
+  log_val_generations: 0
+  rollout_data_dir: /fs/scratch/PAS3151/jluo/exps/rollout/MatchTIR/MatchTIR-4B-extra-thinkback-toolfix1
+  validation_data_dir: /fs/scratch/PAS3151/jluo/exps/validation/MatchTIR/MatchTIR-4B-extra-thinkback-toolfix1
+  nnodes: 1
+  n_gpus_per_node: 4
+  save_freq: 4
+  resume_mode: auto
+  resume_from_path: null
+  val_before_train: true
+  test_freq: 4
+  critic_warmup: 0
+  default_hdfs_dir: null
+  del_local_ckpt_after_load: false
+  default_local_dir: /fs/scratch/PAS3151/jluo/exps/ckpts/MatchTIR/MatchTIR-4B-extra-thinkback-toolfix1
+  max_actor_ckpt_to_keep: null
+  max_critic_ckpt_to_keep: null
+  ray_wait_register_center_timeout: 300
+  val_only: false
+ray_init:
+  num_cpus: 8