| |
| data: |
| train_files: ${oc.env:ORARL_TRAIN_DATA} |
| val_files: ${oc.env:ORARL_VAL_DATA} |
| prompt_key: problem |
| answer_key: answer |
| image_key: images |
| video_key: videos |
| image_dir: ${oc.env:ORARL_MEDIA_ROOT,null} |
| dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4} |
| video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432} |
| video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096} |
| video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128} |
| video_fps: ${oc.env:ORARL_VIDEO_FPS,2} |
| video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608} |
| min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096} |
| image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576} |
| use_preprocessed_videos: false |
| video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only} |
| preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null} |
| inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false} |
| max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,24576} |
| max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096} |
| rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64} |
| mini_rollout_batch_size: null |
| val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16} |
| format_prompt: null |
| override_chat_template: null |
| enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false} |
| response_prefix: "" |
| shuffle: true |
| seed: ${oc.env:ORARL_SEED,42} |
| group_by_task: true |
| group_by_task_key: problem_type |
| filter_overlong_prompts: false |
|
|
| algorithm: |
| name: orarl |
| adv_estimator: grpo |
| scale_rewards: false |
| oracle_injection: true |
| oracle_injection_mode: append |
| oracle_builder: orarl.rewards:build_oracle_response_from_ground_truth |
| oracle_log_exclude: true |
| directional_gain: true |
| directional_gain_gamma: 0.25 |
| directional_gain_positive_only: true |
| directional_gain_recenter: true |
| detached_oracle_advantage: true |
| detached_oracle_advantage_scale: 2.0 |
| detached_oracle_use_directional_gain: false |
| detached_oracle_match_best_ratio: 1.2 |
| detached_oracle_match_best_min: 0.05 |
| detached_oracle_match_best_max: 1.0 |
| oracle_reward_gate_beta: 2.0 |
| selection_prune_ratio: 0.5 |
| selection_keep_oracle: true |
| selection_positive_quota: 1 |
| selection_negative_quota: 2 |
| selection_strict_sign_balance: true |
| post_selection_recenter: true |
| post_selection_rms_match: true |
| post_selection_rms_min_scale: 0.25 |
| disable_kl: true |
| use_kl_loss: false |
| kl_penalty: low_var_kl |
| kl_coef: 0.0 |
|
|
| worker: |
| actor: |
| global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64} |
| micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,1} |
| micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1} |
| ppo_epochs: 1 |
| max_grad_norm: 1.0 |
| clip_ratio_low: 0.2 |
| clip_ratio_high: 0.2 |
| loss_avg_mode: token |
| padding_free: true |
| dynamic_batching: true |
| max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,32768} |
| ulysses_size: 1 |
| use_torch_compile: false |
| model: |
| model_path: ${oc.env:ORARL_MODEL_PATH} |
| tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}} |
| enable_gradient_checkpointing: true |
| trust_remote_code: false |
| freeze_vision_tower: true |
| train_vision_merger: false |
| optim: |
| lr: ${oc.env:ORARL_LEARNING_RATE,1.0e-6} |
| weight_decay: 0.0 |
| strategy: adamw |
| lr_warmup_ratio: 0.0 |
| lr_scheduler_type: constant |
| fsdp: |
| torch_dtype: bf16 |
| enable_full_shard: true |
| enable_cpu_offload: false |
| enable_rank0_init: true |
| use_orig_params: false |
| offload: |
| offload_params: false |
| offload_optimizer: false |
|
|
| rollout: |
| name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm} |
| n: 8 |
| seed: ${oc.env:ORARL_SEED,42} |
| temperature: 1.0 |
| top_p: 0.85 |
| limit_images: 10 |
| gpu_memory_utilization: 0.45 |
| enforce_eager: false |
| enable_chunked_prefill: true |
| tensor_parallel_size: 1 |
| max_num_batched_tokens: 32768 |
| disable_tqdm: true |
| calculate_log_probs: false |
| collect_seq_logprob_for_filter: false |
| val_override_config: |
| temperature: 0.7 |
| top_p: 0.95 |
| n: 1 |
|
|
| ref: |
| fsdp: |
| torch_dtype: bf16 |
| enable_full_shard: true |
| enable_cpu_offload: false |
| enable_rank0_init: true |
| offload: |
| offload_params: false |
|
|
| reward: |
| reward_function: orarl.rewards:compute_score |
| reward_function_kwargs: {} |
|
|
| trainer: |
| total_epochs: 1 |
| max_steps: null |
| project_name: OraRL |
| experiment_name: orarl-9b |
| logger: ["console"] |
| nnodes: 1 |
| n_gpus_per_node: 8 |
| val_freq: -1 |
| val_before_train: false |
| val_only: false |
| val_generations_to_log: 0 |
| save_freq: 50 |
| save_limit: 3 |
| save_model_only: false |
| keep_optim_only_latest: true |
| save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR} |
| load_checkpoint_path: null |
| find_last_checkpoint: false |
| keep_best_train_ckpt: false |
| best_train_metric_key: reward/overall |
| best_train_smooth_window: 5 |
| best_train_min_step: 10 |
|
|
