Add panel_counter_v30_grpo adapter (overnight v31 SFT run + backfill v29/v30 GRPO)

sld_27.0/panel_counter_v30_grpo/adapter_config.json

@@ -0,0 +1,38 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/home/node/.cache/huggingface/hub/models--Qwen--Qwen3.5-9B/snapshots/c202236235762e1c871ad0ccb60c8ee5ba337b9a",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "^(model\\.language_model(?=\\.).*\\.(q_proj|v_proj|k_proj|in_proj_b|in_proj_qkv|out_proj|down_proj|gate_proj|up_proj|in_proj_a|in_proj_z|o_proj))$",
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

sld_27.0/panel_counter_v30_grpo/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1831e055fe961ef3531801ea91bd5c7522e9ada79f811931a3642dc907a740a3
+size 692529024

sld_27.0/panel_counter_v30_grpo/additional_config.json

@@ -0,0 +1 @@
+{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06}

sld_27.0/panel_counter_v30_grpo/args.json

@@ -0,0 +1,478 @@
+{
+  "output_dir": "/workspace/training/checkpoints/panel_counter_v30_grpo/v4-20260726-123127",
+  "per_device_train_batch_size": 1,
+  "num_train_epochs": 100.0,
+  "max_steps": 300,
+  "learning_rate": 5e-06,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 8,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 1,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "wandb"
+  ],
+  "run_name": "/workspace/training/checkpoints/panel_counter_v30_grpo/v4-20260726-123127",
+  "project": "huggingface",
+  "trackio_space_id": "trackio",
+  "eval_strategy": "no",
+  "eval_steps": 5.0,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 1,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": false,
+  "save_strategy": "steps",
+  "save_steps": 5.0,
+  "save_on_each_node": false,
+  "save_total_limit": 8,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": false,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_backend": null,
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": null,
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": "/workspace/training/checkpoints/panel_counter_v30_grpo/v3-20260726-083127/checkpoint-60",
+  "warmup_ratio": null,
+  "logging_dir": "/workspace/training/checkpoints/panel_counter_v30_grpo/v4-20260726-123127/runs",
+  "local_rank": -1,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": null,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": "grpo",
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "lora",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "/home/node/.cache/huggingface/hub/models--Qwen--Qwen3.5-9B/snapshots/c202236235762e1c871ad0ccb60c8ee5ba337b9a",
+  "model_type": "qwen3_5",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen3_5",
+  "system": null,
+  "max_length": 262144,
+  "truncation_strategy": "left",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "last_round",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/workspace/training/grpo_v30_train.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 1,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 4096,
+  "temperature": 0.9,
+  "top_k": 0,
+  "top_p": 0.95,
+  "repetition_penalty": 1.0,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [
+    "/workspace/training/rewards/multi_component_v30.py"
+  ],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": true,
+  "use_hf": true,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 64,
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "reward_model": null,
+  "reward_adapters": [],
+  "reward_model_type": null,
+  "reward_model_revision": null,
+  "reward_template": null,
+  "num_ppo_epochs": 4,
+  "whiten_rewards": false,
+  "kl_coef": 0.05,
+  "cliprange": 0.2,
+  "vf_coef": 0.1,
+  "cliprange_value": 0.2,
+  "gamma": 1.0,
+  "lam": 0.95,
+  "num_mini_batches": 1,
+  "local_rollout_forward_batch_size": 64,
+  "num_sample_generations": 10,
+  "response_length": 4096,
+  "missing_eos_penalty": null,
+  "vllm_gpu_memory_utilization": 0.9,
+  "vllm_tensor_parallel_size": 1,
+  "vllm_pipeline_parallel_size": 1,
+  "vllm_enable_expert_parallel": false,
+  "vllm_max_num_seqs": null,
+  "vllm_max_model_len": null,
+  "vllm_disable_custom_all_reduce": true,
+  "vllm_enforce_eager": false,
+  "vllm_limit_mm_per_prompt": null,
+  "vllm_max_lora_rank": 16,
+  "vllm_enable_prefix_caching": true,
+  "vllm_use_async_engine": null,
+  "vllm_quantization": null,
+  "vllm_reasoning_parser": null,
+  "vllm_disable_cascade_attn": false,
+  "vllm_mm_processor_cache_gb": null,
+  "vllm_speculative_config": null,
+  "vllm_engine_kwargs": {},
+  "vllm_data_parallel_size": 1,
+  "use_vllm": false,
+  "vllm_mode": null,
+  "vllm_enable_lora": false,
+  "vllm_server_base_url": null,
+  "vllm_server_host": null,
+  "vllm_server_port": [
+    8000
+  ],
+  "vllm_server_timeout": 240.0,
+  "vllm_server_group_port": null,
+  "enable_flattened_weight_sync": true,
+  "async_generate": false,
+  "sleep_level": 0,
+  "move_model_batches": null,
+  "offload_optimizer": false,
+  "offload_model": false,
+  "wandb_log_unique_prompts": null,
+  "epsilon": 0.2,
+  "epsilon_high": null,
+  "delta": null,
+  "cosine_min_len_value_wrong": -0.5,
+  "cosine_max_len_value_wrong": 0.0,
+  "cosine_min_len_value_correct": 1.0,
+  "cosine_max_len_value_correct": 0.5,
+  "cosine_max_len": null,
+  "repetition_n_grams": 3,
+  "repetition_max_penalty": -1.0,
+  "reward_model_plugin": null,
+  "chord_sft_dataset": [],
+  "chord_sft_per_device_train_batch_size": null,
+  "chord_enable_phi_function": false,
+  "chord_mu_warmup_steps": null,
+  "chord_mu_decay_steps": null,
+  "chord_mu_peak": null,
+  "chord_mu_valley": null,
+  "sync_ref_model": false,
+  "ref_model_sync_steps": 512,
+  "ref_model_mixup_alpha": 0.6,
+  "multi_turn_scheduler": null,
+  "max_turns": null,
+  "completion_length_limit_scope": "per_round",
+  "vllm_server_pass_dataset": false,
+  "dynamic_sample": false,
+  "max_resample_times": 3,
+  "overlong_filter": false,
+  "soft_max_length": null,
+  "soft_cache_length": null,
+  "scale_rewards": "group",
+  "log_entropy": false,
+  "top_entropy_quantile": 1.0,
+  "importance_sampling_level": "token",
+  "tau_pos": 1.0,
+  "tau_neg": 1.05,
+  "advantage_estimator": "grpo",
+  "kl_in_reward": false,
+  "real_tau": 0.5,
+  "generation_batch_size": null,
+  "steps_per_generation": null,
+  "num_generations_eval": null,
+  "rollout_importance_sampling_mode": null,
+  "rollout_importance_sampling_threshold": 2.0,
+  "log_rollout_offpolicy_metrics": false,
+  "off_policy_sequence_mask_delta": null,
+  "num_generations": 8,
+  "reward_funcs": [
+    "multi_component_v30"
+  ],
+  "reward_weights": null,
+  "log_completions": false,
+  "num_iterations": 1,
+  "teacher_model": null,
+  "teacher_adapters": [],
+  "teacher_model_type": null,
+  "teacher_model_revision": null,
+  "teacher_deepspeed": null,
+  "teacher_model_server": null,
+  "rlhf_type": "grpo",
+  "ref_model": null,
+  "ref_adapters": [],
+  "ref_model_type": null,
+  "ref_model_revision": null,
+  "beta": 0.04,
+  "label_smoothing": 0,
+  "max_completion_length": 4096,
+  "rpo_alpha": null,
+  "ld_alpha": null,
+  "discopop_tau": 0.05,
+  "loss_weights": null,
+  "cpo_alpha": 1.0,
+  "simpo_gamma": 1,
+  "desirable_weight": 1.0,
+  "undesirable_weight": 1.0,
+  "center_rewards_coefficient": null,
+  "sft_alpha": 0,
+  "lmbda": 0.5,
+  "seq_kd": false,
+  "gkd_logits_topk": null,
+  "offload_teacher_model": false,
+  "swift_version": "4.1.0",
+  "ckpt_dir": null,
+  "rank": -1,
+  "global_world_size": 1,
+  "local_world_size": 1,
+  "model_suffix": "Qwen3.5-9B",
+  "model_info": "ModelInfo(model_type='qwen3_5', model_dir='/home/node/.cache/huggingface/hub/models--Qwen--Qwen3.5-9B/snapshots/c202236235762e1c871ad0ccb60c8ee5ba337b9a', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen3_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-0.8B', hf_model_id='Qwen/Qwen3.5-0.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B', hf_model_id='Qwen/Qwen3.5-2B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B', hf_model_id='Qwen/Qwen3.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B', hf_model_id='Qwen/Qwen3.5-9B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B', hf_model_id='Qwen/Qwen3.5-27B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B-FP8', hf_model_id='Qwen/Qwen3.5-27B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-0.8B-Base', hf_model_id='Qwen/Qwen3.5-0.8B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B-Base', hf_model_id='Qwen/Qwen3.5-2B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B-Base', hf_model_id='Qwen/Qwen3.5-4B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B-Base', hf_model_id='Qwen/Qwen3.5-9B-Base', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.models.qwen.Qwen3_5Loader'>, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), mcore_model_type=None, architectures=['Qwen3_5ForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.0.0.dev', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])",
+  "model_dir": "/home/node/.cache/huggingface/hub/models--Qwen--Qwen3.5-9B/snapshots/c202236235762e1c871ad0ccb60c8ee5ba337b9a",
+  "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.templates.qwen.Qwen3_5Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='<think>\\n', non_thinking_prefix='<think>\\n\\n</think>\\n\\n', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "GRPOConfig(output_dir='/workspace/training/checkpoints/panel_counter_v30_grpo/v4-20260726-123127', per_device_train_batch_size=1, num_train_epochs=100.0, max_steps=300, learning_rate=5e-06, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=8, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=1, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['wandb'], run_name='/workspace/training/checkpoints/panel_counter_v30_grpo/v4-20260726-123127', project='huggingface', trackio_space_id='trackio', eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=5.0, eval_delay=0, per_device_eval_batch_size=1, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=False, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=5, save_on_each_node=False, save_total_limit=8, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=True, dataloader_num_workers=1, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_backend=None, ddp_timeout=18000000, fsdp=[], fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, deepspeed=None, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint='/workspace/training/checkpoints/panel_counter_v30_grpo/v3-20260726-083127/checkpoint-60', warmup_ratio=None, logging_dir='/workspace/training/checkpoints/panel_counter_v30_grpo/v4-20260726-123127/runs', local_rank=-1, model_init_kwargs=None, disable_dropout=False, cast_lm_head_to_fp32=False, num_generations=8, num_generations_eval=None, max_completion_length=4096, ds3_gather_for_generation=True, shuffle_dataset=True, generation_batch_size=8, steps_per_generation=8, temperature=0.9, top_p=0.95, top_k=0, min_p=None, generation_kwargs=None, chat_template_kwargs=None, repetition_penalty=1.0, use_transformers_paged=False, cache_implementation=None, use_vllm=False, vllm_mode=None, vllm_model_impl='vllm', vllm_enable_sleep_mode=False, vllm_structured_outputs_regex=None, vllm_server_base_url=None, vllm_server_host=None, vllm_server_port=[8000], vllm_server_timeout=240.0, vllm_group_port=51216, vllm_gpu_memory_utilization=0.9, vllm_max_model_length=None, vllm_tensor_parallel_size=1, beta=0.04, num_iterations=1, epsilon=0.2, delta=None, epsilon_high=None, sapo_temperature_neg=1.05, sapo_temperature_pos=1.0, importance_sampling_level='token', reward_weights=None, multi_objective_aggregation='sum_then_normalize', scale_rewards='group', loss_type='grpo', mask_truncated_completions=False, sync_ref_model=False, ref_model_mixup_alpha=0.6, ref_model_sync_steps=512, top_entropy_quantile=1.0, max_tool_calling_iterations=None, vllm_importance_sampling_correction=True, vllm_importance_sampling_mode='sequence_mask', vllm_importance_sampling_cap=3.0, off_policy_mask_threshold=None, use_bias_correction_kl=False, log_completions=False, num_completions_to_print=None, log_unique_prompts=False, log_completions_hub_repo=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, resume_only_model=False, optimizer=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='lora', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False, vllm_pipeline_parallel_size=1, vllm_enable_expert_parallel=False, vllm_max_num_seqs=None, vllm_max_model_len=None, vllm_disable_custom_all_reduce=True, vllm_enforce_eager=False, vllm_limit_mm_per_prompt=None, vllm_max_lora_rank=16, vllm_enable_prefix_caching=True, vllm_use_async_engine=None, vllm_quantization=None, vllm_reasoning_parser=None, vllm_disable_cascade_attn=False, vllm_mm_processor_cache_gb=None, vllm_speculative_config=None, vllm_engine_kwargs={}, vllm_data_parallel_size=1, stop_words=[], vllm_enable_lora=False, lora_rank=64, vllm_server_group_port=None, enable_flattened_weight_sync=True, async_generate=False, structured_outputs_regex=None, sleep_level=0, move_model_batches=None, offload_optimizer=False, offload_model=False, wandb_log_unique_prompts=None, cosine_min_len_value_wrong=-0.5, cosine_max_len_value_wrong=0.0, cosine_min_len_value_correct=1.0, cosine_max_len_value_correct=0.5, cosine_max_len=4096, repetition_n_grams=3, repetition_max_penalty=-1.0, reward_model=None, reward_model_plugin=None, chord_sft_dataset=[], chord_sft_per_device_train_batch_size=None, chord_enable_phi_function=False, chord_mu_warmup_steps=None, chord_mu_decay_steps=None, chord_mu_peak=None, chord_mu_valley=None, multi_turn_scheduler=None, max_turns=None, completion_length_limit_scope='per_round', vllm_server_pass_dataset=False, dynamic_sample=False, max_resample_times=3, overlong_filter=False, soft_max_length=None, soft_cache_length=None, log_entropy=False, tau_pos=1.0, tau_neg=1.05, advantage_estimator='grpo', kl_in_reward=False, real_tau=0.5, dataset_shuffle=True, rollout_importance_sampling_mode=None, rollout_importance_sampling_threshold=2.0, log_rollout_offpolicy_metrics=False, off_policy_sequence_mask_delta=None)"
+}

sld_27.0/panel_counter_v30_grpo/trainer_state.json

@@ -0,0 +1,1644 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1545253863134658,
+  "eval_steps": 5.0,
+  "global_step": 70,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.004536680760793388,
+      "clip_ratio/high_mean": 0.004536680760793388,
+      "clip_ratio/low_mean": 0.0015869140625,
+      "clip_ratio/low_min": 0.0015869140625,
+      "clip_ratio/region_mean": 0.006123594823293388,
+      "completions/clipped_ratio": 0.4375,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3392.0625,
+      "completions/min_length": 2304.0,
+      "epoch": 0.002207505518763797,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2225625067949295,
+      "kl": 0.08643059153109789,
+      "learning_rate": 4.999862923413781e-06,
+      "loss": -0.2580137252807617,
+      "reward": 0.261492520570755,
+      "reward_std": 0.2802431732416153,
+      "rewards/MultiComponentReward/mean": 0.261492520570755,
+      "rewards/MultiComponentReward/std": 0.2802431881427765,
+      "step": 1,
+      "step_time": 390.9901975095272
+    },
+    {
+      "clip_ratio/high_max": 0.003319235169328749,
+      "clip_ratio/high_mean": 0.003319235169328749,
+      "clip_ratio/low_mean": 0.00262451171875,
+      "clip_ratio/low_min": 0.00262451171875,
+      "clip_ratio/region_mean": 0.005943746888078749,
+      "completions/clipped_ratio": 0.4375,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3402.375,
+      "completions/min_length": 1867.0,
+      "epoch": 0.004415011037527594,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22726306319236755,
+      "kl": 0.1005152128636837,
+      "learning_rate": 4.999451708687114e-06,
+      "loss": 0.039217181503772736,
+      "reward": 0.40391451120376587,
+      "reward_std": 0.44147755205631256,
+      "rewards/MultiComponentReward/mean": 0.40391451120376587,
+      "rewards/MultiComponentReward/std": 0.44147756695747375,
+      "step": 2,
+      "step_time": 394.55434528784826
+    },
+    {
+      "clip_ratio/high_max": 0.004514989675953984,
+      "clip_ratio/high_mean": 0.004514989675953984,
+      "clip_ratio/low_mean": 0.0029670613585039973,
+      "clip_ratio/low_min": 0.0029670613585039973,
+      "clip_ratio/region_mean": 0.007482051034457982,
+      "completions/clipped_ratio": 0.4375,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3278.5,
+      "completions/min_length": 1459.0,
+      "epoch": 0.006622516556291391,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16903212666511536,
+      "kl": 0.12747556529939175,
+      "learning_rate": 4.998766400914329e-06,
+      "loss": -0.09616396576166153,
+      "reward": 0.297086238861084,
+      "reward_std": 0.3382570892572403,
+      "rewards/MultiComponentReward/mean": 0.297086238861084,
+      "rewards/MultiComponentReward/std": 0.3382570892572403,
+      "step": 3,
+      "step_time": 392.7664968017489
+    },
+    {
+      "clip_ratio/high_max": 0.0036451052874326706,
+      "clip_ratio/high_mean": 0.0036451052874326706,
+      "clip_ratio/low_mean": 0.002154926536604762,
+      "clip_ratio/low_min": 0.002154926536604762,
+      "clip_ratio/region_mean": 0.005800031824037433,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 1885.9375,
+      "completions/min_length": 929.0,
+      "epoch": 0.008830022075055188,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.48977038264274597,
+      "kl": 0.14406978711485863,
+      "learning_rate": 4.997807075247147e-06,
+      "loss": 0.037035875022411346,
+      "reward": 0.5267036557197571,
+      "reward_std": 0.41943617165088654,
+      "rewards/MultiComponentReward/mean": 0.5267036557197571,
+      "rewards/MultiComponentReward/std": 0.41943617165088654,
+      "step": 4,
+      "step_time": 374.04128988832235
+    },
+    {
+      "clip_ratio/high_max": 0.004159891657764092,
+      "clip_ratio/high_mean": 0.004159891657764092,
+      "clip_ratio/low_mean": 0.0026522925472818315,
+      "clip_ratio/low_min": 0.0026522925472818315,
+      "clip_ratio/region_mean": 0.006812184205045924,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2444.1875,
+      "completions/min_length": 323.0,
+      "epoch": 0.011037527593818985,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3324630558490753,
+      "kl": 0.21798806637525558,
+      "learning_rate": 4.9965738368864345e-06,
+      "loss": 0.16825205087661743,
+      "reward": 0.5585993528366089,
+      "reward_std": 0.3808545470237732,
+      "rewards/MultiComponentReward/mean": 0.5585993528366089,
+      "rewards/MultiComponentReward/std": 0.3808545768260956,
+      "step": 5,
+      "step_time": 382.24191090650856
+    },
+    {
+      "clip_ratio/high_max": 0.006519458256661892,
+      "clip_ratio/high_mean": 0.006519458256661892,
+      "clip_ratio/low_mean": 0.002338465943466872,
+      "clip_ratio/low_min": 0.002338465943466872,
+      "clip_ratio/region_mean": 0.008857924200128764,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 2889.5,
+      "completions/mean_length": 1112.1875,
+      "completions/min_length": 220.5,
+      "epoch": 0.013245033112582781,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.889543890953064,
+      "kl": 0.4054506756365299,
+      "learning_rate": 4.9950668210706795e-06,
+      "loss": -0.07186673581600189,
+      "reward": 0.7097792029380798,
+      "reward_std": 0.418330579996109,
+      "rewards/MultiComponentReward/mean": 0.7097792029380798,
+      "rewards/MultiComponentReward/std": 0.418330579996109,
+      "step": 6,
+      "step_time": 265.5935098044574
+    },
+    {
+      "clip_ratio/high_max": 0.0055544201750308275,
+      "clip_ratio/high_mean": 0.0055544201750308275,
+      "clip_ratio/low_mean": 0.0023781133349984884,
+      "clip_ratio/low_min": 0.0023781133349984884,
+      "clip_ratio/region_mean": 0.007932533510029316,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2000.3125,
+      "completions/min_length": 329.0,
+      "epoch": 0.01545253863134658,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3339700698852539,
+      "kl": 0.147792749106884,
+      "learning_rate": 4.993286193061145e-06,
+      "loss": 0.17133688926696777,
+      "reward": 0.7523816227912903,
+      "reward_std": 0.24233994632959366,
+      "rewards/MultiComponentReward/mean": 0.7523816227912903,
+      "rewards/MultiComponentReward/std": 0.24233996123075485,
+      "step": 7,
+      "step_time": 374.42519377637655
+    },
+    {
+      "clip_ratio/high_max": 0.00248914523399435,
+      "clip_ratio/high_mean": 0.00248914523399435,
+      "clip_ratio/low_mean": 0.00177001953125,
+      "clip_ratio/low_min": 0.00177001953125,
+      "clip_ratio/region_mean": 0.00425916476524435,
+      "completions/clipped_ratio": 0.3125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2669.25,
+      "completions/min_length": 520.0,
+      "epoch": 0.017660044150110375,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2704744040966034,
+      "kl": 0.10282042529433966,
+      "learning_rate": 4.9912321481237616e-06,
+      "loss": 0.45126625895500183,
+      "reward": 0.4577764421701431,
+      "reward_std": 0.33944071829319,
+      "rewards/MultiComponentReward/mean": 0.4577764421701431,
+      "rewards/MultiComponentReward/std": 0.3394407480955124,
+      "step": 8,
+      "step_time": 388.1322461529635
+    },
+    {
+      "clip_ratio/high_max": 0.003738714032806456,
+      "clip_ratio/high_mean": 0.003738714032806456,
+      "clip_ratio/low_mean": 0.0016605786804575473,
+      "clip_ratio/low_min": 0.0016605786804575473,
+      "clip_ratio/region_mean": 0.005399292713264003,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2970.0,
+      "completions/mean_length": 1617.9375,
+      "completions/min_length": 229.0,
+      "epoch": 0.019867549668874173,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.48584485054016113,
+      "kl": 0.3705522492527962,
+      "learning_rate": 4.9889049115077e-06,
+      "loss": 0.35195228457450867,
+      "reward": 0.6869203746318817,
+      "reward_std": 0.36256058514118195,
+      "rewards/MultiComponentReward/mean": 0.6869203746318817,
+      "rewards/MultiComponentReward/std": 0.36256058514118195,
+      "step": 9,
+      "step_time": 280.1738434997387
+    },
+    {
+      "clip_ratio/high_max": 0.005708650394808501,
+      "clip_ratio/high_mean": 0.005708650394808501,
+      "clip_ratio/low_mean": 0.0012814859510399401,
+      "clip_ratio/low_min": 0.0012814859510399401,
+      "clip_ratio/region_mean": 0.006990136345848441,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3145.0,
+      "completions/mean_length": 1676.5,
+      "completions/min_length": 832.5,
+      "epoch": 0.02207505518763797,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2790386974811554,
+      "kl": 0.16368939075618982,
+      "learning_rate": 4.986304738420684e-06,
+      "loss": -0.07350219041109085,
+      "reward": 0.7483305633068085,
+      "reward_std": 0.3742331862449646,
+      "rewards/MultiComponentReward/mean": 0.7483305633068085,
+      "rewards/MultiComponentReward/std": 0.3742332011461258,
+      "step": 10,
+      "step_time": 291.7711451756768
+    },
+    {
+      "clip_ratio/high_max": 0.0051373289024923,
+      "clip_ratio/high_mean": 0.0051373289024923,
+      "clip_ratio/low_mean": 0.00177001953125,
+      "clip_ratio/low_min": 0.00177001953125,
+      "clip_ratio/region_mean": 0.0069073484337423,
+      "completions/clipped_ratio": 0.5625,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3322.4375,
+      "completions/min_length": 1041.0,
+      "epoch": 0.024282560706401765,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.27244582772254944,
+      "kl": 0.18766336888074875,
+      "learning_rate": 4.983431914000991e-06,
+      "loss": -0.3657270073890686,
+      "reward": 0.2591381222009659,
+      "reward_std": 0.33292441070079803,
+      "rewards/MultiComponentReward/mean": 0.2591381222009659,
+      "rewards/MultiComponentReward/std": 0.33292441070079803,
+      "step": 11,
+      "step_time": 396.88911640644073
+    },
+    {
+      "clip_ratio/high_max": 0.0042414102936163545,
+      "clip_ratio/high_mean": 0.0042414102936163545,
+      "clip_ratio/low_mean": 0.001209420181112364,
+      "clip_ratio/low_min": 0.001209420181112364,
+      "clip_ratio/region_mean": 0.005450830474728718,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2246.375,
+      "completions/min_length": 369.5,
+      "epoch": 0.026490066225165563,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.31503814458847046,
+      "kl": 0.23613896314054728,
+      "learning_rate": 4.980286753286196e-06,
+      "loss": -0.06420311331748962,
+      "reward": 0.5957305431365967,
+      "reward_std": 0.3936467617750168,
+      "rewards/MultiComponentReward/mean": 0.5957305431365967,
+      "rewards/MultiComponentReward/std": 0.393646776676178,
+      "step": 12,
+      "step_time": 378.5018015271053
+    },
+    {
+      "clip_ratio/high_max": 0.006219029513886198,
+      "clip_ratio/high_mean": 0.006219029513886198,
+      "clip_ratio/low_mean": 0.00124887510901317,
+      "clip_ratio/low_min": 0.00124887510901317,
+      "clip_ratio/region_mean": 0.007467904622899368,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 3422.5,
+      "completions/mean_length": 2187.6875,
+      "completions/min_length": 537.5,
+      "epoch": 0.02869757174392936,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4874683618545532,
+      "kl": 0.1610732921399176,
+      "learning_rate": 4.9768696011786095e-06,
+      "loss": -0.14555585384368896,
+      "reward": 0.7568104565143585,
+      "reward_std": 0.24511048570275307,
+      "rewards/MultiComponentReward/mean": 0.7568104565143585,
+      "rewards/MultiComponentReward/std": 0.24511050432920456,
+      "step": 13,
+      "step_time": 314.2461526854895
+    },
+    {
+      "clip_ratio/high_max": 0.002680988982319832,
+      "clip_ratio/high_mean": 0.002680988982319832,
+      "clip_ratio/low_mean": 0.0023402327205985785,
+      "clip_ratio/low_min": 0.0023402327205985785,
+      "clip_ratio/region_mean": 0.00502122170291841,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3919.5,
+      "completions/mean_length": 2166.625,
+      "completions/min_length": 862.0,
+      "epoch": 0.03090507726269316,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3764588534832001,
+      "kl": 0.31136148795485497,
+      "learning_rate": 4.973180832407471e-06,
+      "loss": -0.08120034635066986,
+      "reward": 0.40853719413280487,
+      "reward_std": 0.18009745329618454,
+      "rewards/MultiComponentReward/mean": 0.40853719413280487,
+      "rewards/MultiComponentReward/std": 0.18009746074676514,
+      "step": 14,
+      "step_time": 354.3425365053117
+    },
+    {
+      "clip_ratio/high_max": 0.006010087672621012,
+      "clip_ratio/high_mean": 0.006010087672621012,
+      "clip_ratio/low_mean": 0.001259958662558347,
+      "clip_ratio/low_min": 0.001259958662558347,
+      "clip_ratio/region_mean": 0.007270046335179359,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3298.5,
+      "completions/mean_length": 1722.125,
+      "completions/min_length": 1068.5,
+      "epoch": 0.033112582781456956,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.34743329882621765,
+      "kl": 0.17298532463610172,
+      "learning_rate": 4.9692208514878445e-06,
+      "loss": -0.5539411902427673,
+      "reward": 0.8423767983913422,
+      "reward_std": 0.061326997354626656,
+      "rewards/MultiComponentReward/mean": 0.8423767983913422,
+      "rewards/MultiComponentReward/std": 0.061326999217271805,
+      "step": 15,
+      "step_time": 305.30156579380855
+    },
+    {
+      "clip_ratio/high_max": 0.006434470997191966,
+      "clip_ratio/high_mean": 0.006434470997191966,
+      "clip_ratio/low_mean": 0.00152587890625,
+      "clip_ratio/low_min": 0.00152587890625,
+      "clip_ratio/region_mean": 0.007960349903441966,
+      "completions/clipped_ratio": 0.3125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3167.4375,
+      "completions/min_length": 1716.5,
+      "epoch": 0.03532008830022075,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20621593296527863,
+      "kl": 0.1370371589437127,
+      "learning_rate": 4.964990092676263e-06,
+      "loss": -0.31487035751342773,
+      "reward": 0.5214030742645264,
+      "reward_std": 0.4230431914329529,
+      "rewards/MultiComponentReward/mean": 0.5214030742645264,
+      "rewards/MultiComponentReward/std": 0.42304322123527527,
+      "step": 16,
+      "step_time": 384.0548590766266
+    },
+    {
+      "clip_ratio/high_max": 0.0075649446225725114,
+      "clip_ratio/high_mean": 0.0075649446225725114,
+      "clip_ratio/low_mean": 0.00164794921875,
+      "clip_ratio/low_min": 0.00164794921875,
+      "clip_ratio/region_mean": 0.009212893841322511,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2964.75,
+      "completions/min_length": 1122.0,
+      "epoch": 0.037527593818984545,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.337228387594223,
+      "kl": 0.20724927634000778,
+      "learning_rate": 4.960489019923105e-06,
+      "loss": 0.09976287186145782,
+      "reward": 0.6003734767436981,
+      "reward_std": 0.35703691840171814,
+      "rewards/MultiComponentReward/mean": 0.6003734767436981,
+      "rewards/MultiComponentReward/std": 0.35703691840171814,
+      "step": 17,
+      "step_time": 377.73133220197633
+    },
+    {
+      "clip_ratio/high_max": 0.0014927737356629223,
+      "clip_ratio/high_mean": 0.0014927737356629223,
+      "clip_ratio/low_mean": 0.003204345703125,
+      "clip_ratio/low_min": 0.003204345703125,
+      "clip_ratio/region_mean": 0.004697119438787922,
+      "completions/clipped_ratio": 0.625,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3403.4375,
+      "completions/min_length": 1413.0,
+      "epoch": 0.039735099337748346,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3440120220184326,
+      "kl": 0.15303738554939628,
+      "learning_rate": 4.9557181268217225e-06,
+      "loss": 0.08143772184848785,
+      "reward": 0.24983403086662292,
+      "reward_std": 0.39776015281677246,
+      "rewards/MultiComponentReward/mean": 0.24983403086662292,
+      "rewards/MultiComponentReward/std": 0.3977601230144501,
+      "step": 18,
+      "step_time": 388.1805465449579
+    },
+    {
+      "clip_ratio/high_max": 0.005191743664909154,
+      "clip_ratio/high_mean": 0.005191743664909154,
+      "clip_ratio/low_mean": 0.000708264677086845,
+      "clip_ratio/low_min": 0.000708264677086845,
+      "clip_ratio/region_mean": 0.005900008341995999,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 3715.5,
+      "completions/mean_length": 2667.375,
+      "completions/min_length": 1683.0,
+      "epoch": 0.04194260485651214,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22380870580673218,
+      "kl": 0.12537722196429968,
+      "learning_rate": 4.9506779365543054e-06,
+      "loss": -0.3121515214443207,
+      "reward": 0.7294442653656006,
+      "reward_std": 0.23573941737413406,
+      "rewards/MultiComponentReward/mean": 0.7294442653656006,
+      "rewards/MultiComponentReward/std": 0.2357394192367792,
+      "step": 19,
+      "step_time": 349.45764507493004
+    },
+    {
+      "clip_ratio/high_max": 0.004706757143139839,
+      "clip_ratio/high_mean": 0.004706757143139839,
+      "clip_ratio/low_mean": 0.0014424767578020692,
+      "clip_ratio/low_min": 0.0014424767578020692,
+      "clip_ratio/region_mean": 0.006149233900941908,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3183.0,
+      "completions/mean_length": 1463.875,
+      "completions/min_length": 873.5,
+      "epoch": 0.04415011037527594,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6936144232749939,
+      "kl": 0.16285017970949411,
+      "learning_rate": 4.9453690018345144e-06,
+      "loss": -0.0027779489755630493,
+      "reward": 0.8973818123340607,
+      "reward_std": 0.14456791430711746,
+      "rewards/MultiComponentReward/mean": 0.8973818123340607,
+      "rewards/MultiComponentReward/std": 0.14456793665885925,
+      "step": 20,
+      "step_time": 289.136493598111
+    },
+    {
+      "clip_ratio/high_max": 0.008684041909873486,
+      "clip_ratio/high_mean": 0.008684041909873486,
+      "clip_ratio/low_mean": 0.0015138110611587763,
+      "clip_ratio/low_min": 0.0015138110611587763,
+      "clip_ratio/region_mean": 0.010197852971032262,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2634.5,
+      "completions/mean_length": 1312.625,
+      "completions/min_length": 716.5,
+      "epoch": 0.046357615894039736,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3519524335861206,
+      "kl": 0.19825585931539536,
+      "learning_rate": 4.939791904846869e-06,
+      "loss": 0.2327621430158615,
+      "reward": 0.9032591879367828,
+      "reward_std": 0.13960547745227814,
+      "rewards/MultiComponentReward/mean": 0.9032591879367828,
+      "rewards/MultiComponentReward/std": 0.13960548862814903,
+      "step": 21,
+      "step_time": 246.427539577242
+    },
+    {
+      "clip_ratio/high_max": 0.0024823983549140394,
+      "clip_ratio/high_mean": 0.0024823983549140394,
+      "clip_ratio/low_mean": 0.002288818359375,
+      "clip_ratio/low_min": 0.002288818359375,
+      "clip_ratio/region_mean": 0.004771216714289039,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3127.875,
+      "completions/min_length": 1410.0,
+      "epoch": 0.04856512141280353,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.33166858553886414,
+      "kl": 0.1960495300590992,
+      "learning_rate": 4.933947257182901e-06,
+      "loss": -0.009233146905899048,
+      "reward": 0.4366578757762909,
+      "reward_std": 0.45074263215065,
+      "rewards/MultiComponentReward/mean": 0.4366578757762909,
+      "rewards/MultiComponentReward/std": 0.4507426619529724,
+      "step": 22,
+      "step_time": 390.3793193195015
+    },
+    {
+      "clip_ratio/high_max": 0.004601968394126743,
+      "clip_ratio/high_mean": 0.004601968394126743,
+      "clip_ratio/low_mean": 0.0022085192031227052,
+      "clip_ratio/low_min": 0.0022085192031227052,
+      "clip_ratio/region_mean": 0.006810487597249448,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2988.0,
+      "completions/mean_length": 1626.125,
+      "completions/min_length": 913.0,
+      "epoch": 0.05077262693156733,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6514219641685486,
+      "kl": 0.14153485326096416,
+      "learning_rate": 4.92783569977409e-06,
+      "loss": 0.3597075939178467,
+      "reward": 0.7424880266189575,
+      "reward_std": 0.3246006518602371,
+      "rewards/MultiComponentReward/mean": 0.7424880266189575,
+      "rewards/MultiComponentReward/std": 0.3246006518602371,
+      "step": 23,
+      "step_time": 274.9694087076932
+    },
+    {
+      "clip_ratio/high_max": 0.0039050524937920272,
+      "clip_ratio/high_mean": 0.0039050524937920272,
+      "clip_ratio/low_mean": 0.0011284862994216383,
+      "clip_ratio/low_min": 0.0011284862994216383,
+      "clip_ratio/region_mean": 0.0050335387932136655,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 1619.875,
+      "completions/min_length": 399.5,
+      "epoch": 0.052980132450331126,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2053840607404709,
+      "kl": 0.20350851770490408,
+      "learning_rate": 4.921457902821578e-06,
+      "loss": -0.011441629379987717,
+      "reward": 0.6857281923294067,
+      "reward_std": 0.28443270921707153,
+      "rewards/MultiComponentReward/mean": 0.6857281923294067,
+      "rewards/MultiComponentReward/std": 0.28443270921707153,
+      "step": 24,
+      "step_time": 366.2704308098182
+    },
+    {
+      "clip_ratio/high_max": 0.006151905807200819,
+      "clip_ratio/high_mean": 0.006151905807200819,
+      "clip_ratio/low_mean": 0.0017167689511552453,
+      "clip_ratio/low_min": 0.0017167689511552453,
+      "clip_ratio/region_mean": 0.007868674758356065,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 4045.5,
+      "completions/mean_length": 2654.1875,
+      "completions/min_length": 991.0,
+      "epoch": 0.05518763796909492,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18877553939819336,
+      "kl": 0.1717006042599678,
+      "learning_rate": 4.914814565722671e-06,
+      "loss": -0.19193655252456665,
+      "reward": 0.7170408964157104,
+      "reward_std": 0.221495708450675,
+      "rewards/MultiComponentReward/mean": 0.7170408964157104,
+      "rewards/MultiComponentReward/std": 0.221495708450675,
+      "step": 25,
+      "step_time": 376.1836411692202
+    },
+    {
+      "clip_ratio/high_max": 0.003862613928504288,
+      "clip_ratio/high_mean": 0.003862613928504288,
+      "clip_ratio/low_mean": 0.0022122596274130046,
+      "clip_ratio/low_min": 0.0022122596274130046,
+      "clip_ratio/region_mean": 0.006074873555917293,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2507.5,
+      "completions/mean_length": 1542.875,
+      "completions/min_length": 1025.0,
+      "epoch": 0.05739514348785872,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20356233417987823,
+      "kl": 0.14280430227518082,
+      "learning_rate": 4.907906416994146e-06,
+      "loss": -0.1313701868057251,
+      "reward": 0.8475965857505798,
+      "reward_std": 0.18501508980989456,
+      "rewards/MultiComponentReward/mean": 0.8475965857505798,
+      "rewards/MultiComponentReward/std": 0.18501508235931396,
+      "step": 26,
+      "step_time": 235.86966329906136
+    },
+    {
+      "clip_ratio/high_max": 0.003541313431924209,
+      "clip_ratio/high_mean": 0.003541313431924209,
+      "clip_ratio/low_mean": 0.00128173828125,
+      "clip_ratio/low_min": 0.00128173828125,
+      "clip_ratio/region_mean": 0.004823051713174209,
+      "completions/clipped_ratio": 0.3125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3276.375,
+      "completions/min_length": 2009.0,
+      "epoch": 0.059602649006622516,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16266532242298126,
+      "kl": 0.09048591833561659,
+      "learning_rate": 4.900734214192358e-06,
+      "loss": -0.22580142319202423,
+      "reward": 0.5785741358995438,
+      "reward_std": 0.432532399892807,
+      "rewards/MultiComponentReward/mean": 0.5785741358995438,
+      "rewards/MultiComponentReward/std": 0.4325324296951294,
+      "step": 27,
+      "step_time": 391.1576498737559
+    },
+    {
+      "clip_ratio/high_max": 0.003771795134525746,
+      "clip_ratio/high_mean": 0.003771795134525746,
+      "clip_ratio/low_mean": 0.002044677734375,
+      "clip_ratio/low_min": 0.002044677734375,
+      "clip_ratio/region_mean": 0.005816472868900746,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3543.4375,
+      "completions/min_length": 2016.0,
+      "epoch": 0.06181015452538632,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.23085403442382812,
+      "kl": 0.11726451432332397,
+      "learning_rate": 4.893298743830168e-06,
+      "loss": 0.011457636952400208,
+      "reward": 0.39609572291374207,
+      "reward_std": 0.4118330776691437,
+      "rewards/MultiComponentReward/mean": 0.39609572291374207,
+      "rewards/MultiComponentReward/std": 0.4118330776691437,
+      "step": 28,
+      "step_time": 397.54813467431813
+    },
+    {
+      "clip_ratio/high_max": 0.004495721834246069,
+      "clip_ratio/high_mean": 0.004495721834246069,
+      "clip_ratio/low_mean": 0.0019099603814538568,
+      "clip_ratio/low_min": 0.0019099603814538568,
+      "clip_ratio/region_mean": 0.006405682215699926,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2369.5,
+      "completions/mean_length": 1519.6875,
+      "completions/min_length": 938.5,
+      "epoch": 0.0640176600441501,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5666957497596741,
+      "kl": 0.13546149153262377,
+      "learning_rate": 4.885600821290692e-06,
+      "loss": 0.06810422241687775,
+      "reward": 0.8998453915119171,
+      "reward_std": 0.10061540827155113,
+      "rewards/MultiComponentReward/mean": 0.8998453915119171,
+      "rewards/MultiComponentReward/std": 0.10061541572213173,
+      "step": 29,
+      "step_time": 225.84905913239345
+    },
+    {
+      "clip_ratio/high_max": 0.0040456526330672204,
+      "clip_ratio/high_mean": 0.0040456526330672204,
+      "clip_ratio/low_mean": 0.0023748770472593606,
+      "clip_ratio/low_min": 0.0023748770472593606,
+      "clip_ratio/region_mean": 0.006420529680326581,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3651.5,
+      "completions/mean_length": 2009.8125,
+      "completions/min_length": 910.0,
+      "epoch": 0.06622516556291391,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3052200675010681,
+      "kl": 0.13525741174817085,
+      "learning_rate": 4.8776412907378845e-06,
+      "loss": 0.31950289011001587,
+      "reward": 0.7715935409069061,
+      "reward_std": 0.23119045048952103,
+      "rewards/MultiComponentReward/mean": 0.7715935409069061,
+      "rewards/MultiComponentReward/std": 0.23119046539068222,
+      "step": 30,
+      "step_time": 337.83108606841415
+    },
+    {
+      "clip_ratio/high_max": 0.002829255477990955,
+      "clip_ratio/high_mean": 0.002829255477990955,
+      "clip_ratio/low_mean": 0.001434326171875,
+      "clip_ratio/low_min": 0.001434326171875,
+      "clip_ratio/region_mean": 0.004263581649865955,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3670.3125,
+      "completions/min_length": 2440.0,
+      "epoch": 0.0684326710816777,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17697112262248993,
+      "kl": 0.08350045792758465,
+      "learning_rate": 4.869421025023965e-06,
+      "loss": -0.005949936807155609,
+      "reward": 0.3625205308198929,
+      "reward_std": 0.38944023847579956,
+      "rewards/MultiComponentReward/mean": 0.3625205308198929,
+      "rewards/MultiComponentReward/std": 0.38944023847579956,
+      "step": 31,
+      "step_time": 389.3736478481442
+    },
+    {
+      "clip_ratio/high_max": 0.0018982566543854773,
+      "clip_ratio/high_mean": 0.0018982566543854773,
+      "clip_ratio/low_mean": 0.00244140625,
+      "clip_ratio/low_min": 0.00244140625,
+      "clip_ratio/region_mean": 0.004339662904385477,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3331.5,
+      "completions/min_length": 2154.5,
+      "epoch": 0.0706401766004415,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.31352514028549194,
+      "kl": 0.08681585732847452,
+      "learning_rate": 4.860940925593703e-06,
+      "loss": 0.1531461924314499,
+      "reward": 0.3806203752756119,
+      "reward_std": 0.45519421994686127,
+      "rewards/MultiComponentReward/mean": 0.3806203752756119,
+      "rewards/MultiComponentReward/std": 0.45519423484802246,
+      "step": 32,
+      "step_time": 391.1217633676715
+    },
+    {
+      "clip_ratio/high_max": 0.00427300637238659,
+      "clip_ratio/high_mean": 0.00427300637238659,
+      "clip_ratio/low_mean": 0.0009272472525481135,
+      "clip_ratio/low_min": 0.0009272472525481135,
+      "clip_ratio/region_mean": 0.005200253624934703,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3600.5,
+      "completions/mean_length": 1683.8125,
+      "completions/min_length": 902.5,
+      "epoch": 0.0728476821192053,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2556222677230835,
+      "kl": 0.14949997700750828,
+      "learning_rate": 4.852201922385564e-06,
+      "loss": -0.13168251514434814,
+      "reward": 0.8324811160564423,
+      "reward_std": 0.2306104227900505,
+      "rewards/MultiComponentReward/mean": 0.8324811160564423,
+      "rewards/MultiComponentReward/std": 0.2306104302406311,
+      "step": 33,
+      "step_time": 321.7230081311427
+    },
+    {
+      "clip_ratio/high_max": 0.003084327036049217,
+      "clip_ratio/high_mean": 0.003084327036049217,
+      "clip_ratio/low_mean": 0.0023715725692454726,
+      "clip_ratio/low_min": 0.0023715725692454726,
+      "clip_ratio/region_mean": 0.0054558996052946895,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2324.0,
+      "completions/mean_length": 1950.6875,
+      "completions/min_length": 1372.5,
+      "epoch": 0.07505518763796909,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22313375771045685,
+      "kl": 0.12166391499340534,
+      "learning_rate": 4.84320497372973e-06,
+      "loss": -0.08645301312208176,
+      "reward": 0.9029195010662079,
+      "reward_std": 0.06034624204039574,
+      "rewards/MultiComponentReward/mean": 0.9029195010662079,
+      "rewards/MultiComponentReward/std": 0.06034623831510544,
+      "step": 34,
+      "step_time": 227.21475145453587
+    },
+    {
+      "clip_ratio/high_max": 0.005887008446734399,
+      "clip_ratio/high_mean": 0.005887008446734399,
+      "clip_ratio/low_mean": 0.00048828125,
+      "clip_ratio/low_min": 0.00048828125,
+      "clip_ratio/region_mean": 0.006375289696734399,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2648.5,
+      "completions/min_length": 893.5,
+      "epoch": 0.0772626931567329,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18673716485500336,
+      "kl": 0.10563921928405762,
+      "learning_rate": 4.833951066243004e-06,
+      "loss": -0.29802197217941284,
+      "reward": 0.6780980825424194,
+      "reward_std": 0.42671698331832886,
+      "rewards/MultiComponentReward/mean": 0.6780980825424194,
+      "rewards/MultiComponentReward/std": 0.42671696841716766,
+      "step": 35,
+      "step_time": 374.7209934145212
+    },
+    {
+      "clip_ratio/high_max": 0.0016851206310093403,
+      "clip_ratio/high_mean": 0.0016851206310093403,
+      "clip_ratio/low_mean": 0.0018310546875,
+      "clip_ratio/low_min": 0.0018310546875,
+      "clip_ratio/region_mean": 0.0035161753185093403,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2422.625,
+      "completions/min_length": 361.5,
+      "epoch": 0.07947019867549669,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22494255006313324,
+      "kl": 0.17482219403609633,
+      "learning_rate": 4.824441214720629e-06,
+      "loss": 0.27602869272232056,
+      "reward": 0.5321091264486313,
+      "reward_std": 0.42589741945266724,
+      "rewards/MultiComponentReward/mean": 0.5321091264486313,
+      "rewards/MultiComponentReward/std": 0.42589743435382843,
+      "step": 36,
+      "step_time": 379.9912438802421
+    },
+    {
+      "clip_ratio/high_max": 0.0027581260656006634,
+      "clip_ratio/high_mean": 0.0027581260656006634,
+      "clip_ratio/low_mean": 0.002960205078125,
+      "clip_ratio/low_min": 0.002960205078125,
+      "clip_ratio/region_mean": 0.005718331143725663,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3481.6875,
+      "completions/min_length": 1714.5,
+      "epoch": 0.08167770419426049,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4158739149570465,
+      "kl": 0.10050415620207787,
+      "learning_rate": 4.814676462024988e-06,
+      "loss": -0.12720987200737,
+      "reward": 0.24663789942860603,
+      "reward_std": 0.36628031730651855,
+      "rewards/MultiComponentReward/mean": 0.24663789942860603,
+      "rewards/MultiComponentReward/std": 0.36628031730651855,
+      "step": 37,
+      "step_time": 389.53164913272485
+    },
+    {
+      "clip_ratio/high_max": 0.0036612136173062027,
+      "clip_ratio/high_mean": 0.0036612136173062027,
+      "clip_ratio/low_mean": 0.0024481196305714548,
+      "clip_ratio/low_min": 0.0024481196305714548,
+      "clip_ratio/region_mean": 0.006109333247877657,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3739.5,
+      "completions/mean_length": 2327.375,
+      "completions/min_length": 280.5,
+      "epoch": 0.08388520971302428,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.413541316986084,
+      "kl": 0.23857311811298132,
+      "learning_rate": 4.804657878971252e-06,
+      "loss": 0.1663297414779663,
+      "reward": 0.8061079978942871,
+      "reward_std": 0.1977117396891117,
+      "rewards/MultiComponentReward/mean": 0.8061079978942871,
+      "rewards/MultiComponentReward/std": 0.1977117545902729,
+      "step": 38,
+      "step_time": 340.510907412041
+    },
+    {
+      "clip_ratio/high_max": 0.009116197703406215,
+      "clip_ratio/high_mean": 0.009116197703406215,
+      "clip_ratio/low_mean": 0.002183556731324643,
+      "clip_ratio/low_min": 0.002183556731324643,
+      "clip_ratio/region_mean": 0.011299754434730858,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3287.0,
+      "completions/mean_length": 1864.0625,
+      "completions/min_length": 583.0,
+      "epoch": 0.08609271523178808,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.27449601888656616,
+      "kl": 0.24706059228628874,
+      "learning_rate": 4.794386564209953e-06,
+      "loss": -0.003756396472454071,
+      "reward": 0.8452397286891937,
+      "reward_std": 0.19490926712751389,
+      "rewards/MultiComponentReward/mean": 0.8452397286891937,
+      "rewards/MultiComponentReward/std": 0.19490928761661053,
+      "step": 39,
+      "step_time": 295.4189805756323
+    },
+    {
+      "clip_ratio/high_max": 0.005593291949480772,
+      "clip_ratio/high_mean": 0.005593291949480772,
+      "clip_ratio/low_mean": 0.001316860259976238,
+      "clip_ratio/low_min": 0.001316860259976238,
+      "clip_ratio/region_mean": 0.00691015220945701,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 3758.0,
+      "completions/mean_length": 2535.5625,
+      "completions/min_length": 1146.5,
+      "epoch": 0.08830022075055188,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.24644321203231812,
+      "kl": 0.12604943895712495,
+      "learning_rate": 4.783863644106502e-06,
+      "loss": -0.016649454832077026,
+      "reward": 0.7764062583446503,
+      "reward_std": 0.2480708546936512,
+      "rewards/MultiComponentReward/mean": 0.7764062583446503,
+      "rewards/MultiComponentReward/std": 0.2480708546936512,
+      "step": 40,
+      "step_time": 349.41997333150357
+    },
+    {
+      "clip_ratio/high_max": 0.006368857342749834,
+      "clip_ratio/high_mean": 0.006368857342749834,
+      "clip_ratio/low_mean": 0.001155435515101999,
+      "clip_ratio/low_min": 0.001155435515101999,
+      "clip_ratio/region_mean": 0.007524292857851833,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1950.5,
+      "completions/mean_length": 1348.625,
+      "completions/min_length": 717.5,
+      "epoch": 0.09050772626931568,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.24919532239437103,
+      "kl": 0.1538570187985897,
+      "learning_rate": 4.773090272617672e-06,
+      "loss": -0.20922431349754333,
+      "reward": 0.9199557602405548,
+      "reward_std": 0.08289439044892788,
+      "rewards/MultiComponentReward/mean": 0.9199557602405548,
+      "rewards/MultiComponentReward/std": 0.08289438113570213,
+      "step": 41,
+      "step_time": 187.90400995500386
+    },
+    {
+      "clip_ratio/high_max": 0.0037301366683095694,
+      "clip_ratio/high_mean": 0.0037301366683095694,
+      "clip_ratio/low_mean": 0.0026621700380928814,
+      "clip_ratio/low_min": 0.0026621700380928814,
+      "clip_ratio/region_mean": 0.006392306706402451,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2435.0625,
+      "completions/min_length": 588.0,
+      "epoch": 0.09271523178807947,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.27514010667800903,
+      "kl": 0.14031154196709394,
+      "learning_rate": 4.762067631165049e-06,
+      "loss": 0.43530726432800293,
+      "reward": 0.6737551689147949,
+      "reward_std": 0.354981392621994,
+      "rewards/MultiComponentReward/mean": 0.6737551689147949,
+      "rewards/MultiComponentReward/std": 0.3549814224243164,
+      "step": 42,
+      "step_time": 378.5200777417049
+    },
+    {
+      "clip_ratio/high_max": 0.0024539779406040907,
+      "clip_ratio/high_mean": 0.0024539779406040907,
+      "clip_ratio/low_mean": 0.002349853515625,
+      "clip_ratio/low_min": 0.002349853515625,
+      "clip_ratio/region_mean": 0.004803831456229091,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3008.875,
+      "completions/min_length": 808.0,
+      "epoch": 0.09492273730684327,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19144803285598755,
+      "kl": 0.19372461177408695,
+      "learning_rate": 4.750796928505484e-06,
+      "loss": -0.014111649245023727,
+      "reward": 0.3934132233262062,
+      "reward_std": 0.37163105607032776,
+      "rewards/MultiComponentReward/mean": 0.3934132233262062,
+      "rewards/MultiComponentReward/std": 0.37163107097148895,
+      "step": 43,
+      "step_time": 382.2734902310185
+    },
+    {
+      "clip_ratio/high_max": 0.003861240576952696,
+      "clip_ratio/high_mean": 0.003861240576952696,
+      "clip_ratio/low_mean": 0.0032202915172092617,
+      "clip_ratio/low_min": 0.0032202915172092617,
+      "clip_ratio/region_mean": 0.0070815320941619575,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 3355.0,
+      "completions/mean_length": 1775.5,
+      "completions/min_length": 320.0,
+      "epoch": 0.09713024282560706,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.42149120569229126,
+      "kl": 0.4061455652117729,
+      "learning_rate": 4.7392794005985324e-06,
+      "loss": 0.14814165234565735,
+      "reward": 0.7460151612758636,
+      "reward_std": 0.23946712166070938,
+      "rewards/MultiComponentReward/mean": 0.7460151612758636,
+      "rewards/MultiComponentReward/std": 0.23946715146303177,
+      "step": 44,
+      "step_time": 302.60926858196035
+    },
+    {
+      "clip_ratio/high_max": 0.00499365694122389,
+      "clip_ratio/high_mean": 0.00499365694122389,
+      "clip_ratio/low_mean": 0.0013380597811192274,
+      "clip_ratio/low_min": 0.0013380597811192274,
+      "clip_ratio/region_mean": 0.006331716722343117,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3011.5,
+      "completions/min_length": 1578.0,
+      "epoch": 0.09933774834437085,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.24550500512123108,
+      "kl": 0.13234132900834084,
+      "learning_rate": 4.72751631047092e-06,
+      "loss": -0.01584070362150669,
+      "reward": 0.3628343641757965,
+      "reward_std": 0.2378975674510002,
+      "rewards/MultiComponentReward/mean": 0.3628343641757965,
+      "rewards/MultiComponentReward/std": 0.237897589802742,
+      "step": 45,
+      "step_time": 381.75905673997477
+    },
+    {
+      "clip_ratio/high_max": 0.004896236292552203,
+      "clip_ratio/high_mean": 0.004896236292552203,
+      "clip_ratio/low_mean": 0.000457763671875,
+      "clip_ratio/low_min": 0.000457763671875,
+      "clip_ratio/region_mean": 0.005353999964427203,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 1583.5625,
+      "completions/min_length": 481.0,
+      "epoch": 0.10154525386313466,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20652994513511658,
+      "kl": 0.2916934099048376,
+      "learning_rate": 4.715508948078037e-06,
+      "loss": -0.03914305195212364,
+      "reward": 0.8016833662986755,
+      "reward_std": 0.3322996646165848,
+      "rewards/MultiComponentReward/mean": 0.8016833662986755,
+      "rewards/MultiComponentReward/std": 0.3322996646165848,
+      "step": 46,
+      "step_time": 357.3734937007539
+    },
+    {
+      "clip_ratio/high_max": 0.007430244877468795,
+      "clip_ratio/high_mean": 0.007430244877468795,
+      "clip_ratio/low_mean": 0.0003662109375,
+      "clip_ratio/low_min": 0.0003662109375,
+      "clip_ratio/region_mean": 0.007796455814968795,
+      "completions/clipped_ratio": 0.3125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2251.0,
+      "completions/min_length": 197.5,
+      "epoch": 0.10375275938189846,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7294849753379822,
+      "kl": 0.21630899980664253,
+      "learning_rate": 4.703258630162481e-06,
+      "loss": -0.49959832429885864,
+      "reward": 0.5816261321306229,
+      "reward_std": 0.40521249175071716,
+      "rewards/MultiComponentReward/mean": 0.5816261321306229,
+      "rewards/MultiComponentReward/std": 0.40521247684955597,
+      "step": 47,
+      "step_time": 360.30742040788755
+    },
+    {
+      "clip_ratio/high_max": 0.0036448093596845865,
+      "clip_ratio/high_mean": 0.0036448093596845865,
+      "clip_ratio/low_mean": 0.0017122222925536335,
+      "clip_ratio/low_min": 0.0017122222925536335,
+      "clip_ratio/region_mean": 0.00535703165223822,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 1619.6875,
+      "completions/min_length": 740.0,
+      "epoch": 0.10596026490066225,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1014658734202385,
+      "kl": 0.12806300446391106,
+      "learning_rate": 4.690766700109659e-06,
+      "loss": -0.19214870035648346,
+      "reward": 0.7811862528324127,
+      "reward_std": 0.3373981714248657,
+      "rewards/MultiComponentReward/mean": 0.7811862528324127,
+      "rewards/MultiComponentReward/std": 0.3373981714248657,
+      "step": 48,
+      "step_time": 353.47038035281
+    },
+    {
+      "clip_ratio/high_max": 0.005789522372651845,
+      "clip_ratio/high_mean": 0.005789522372651845,
+      "clip_ratio/low_mean": 0.001129150390625,
+      "clip_ratio/low_min": 0.001129150390625,
+      "clip_ratio/region_mean": 0.006918672763276845,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 3902.5,
+      "completions/mean_length": 2033.8125,
+      "completions/min_length": 464.0,
+      "epoch": 0.10816777041942605,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1698242723941803,
+      "kl": 0.14896809123456478,
+      "learning_rate": 4.6780345278004744e-06,
+      "loss": 0.1023855209350586,
+      "reward": 0.842141330242157,
+      "reward_std": 0.2511163093149662,
+      "rewards/MultiComponentReward/mean": 0.842141330242157,
+      "rewards/MultiComponentReward/std": 0.2511163130402565,
+      "step": 49,
+      "step_time": 353.20178638910875
+    },
+    {
+      "clip_ratio/high_max": 0.005740969616454095,
+      "clip_ratio/high_mean": 0.005740969616454095,
+      "clip_ratio/low_mean": 0.001129150390625,
+      "clip_ratio/low_min": 0.001129150390625,
+      "clip_ratio/region_mean": 0.006870120007079095,
+      "completions/clipped_ratio": 0.3125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3368.5625,
+      "completions/min_length": 2310.5,
+      "epoch": 0.11037527593818984,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.23211093246936798,
+      "kl": 0.12293548043817282,
+      "learning_rate": 4.665063509461098e-06,
+      "loss": -0.3495488166809082,
+      "reward": 0.47674229741096497,
+      "reward_std": 0.4411779195070267,
+      "rewards/MultiComponentReward/mean": 0.47674229741096497,
+      "rewards/MultiComponentReward/std": 0.44117793440818787,
+      "step": 50,
+      "step_time": 381.0282032000832
+    },
+    {
+      "clip_ratio/high_max": 0.006321625696728006,
+      "clip_ratio/high_mean": 0.006321625696728006,
+      "clip_ratio/low_mean": 0.001495361328125,
+      "clip_ratio/low_min": 0.001495361328125,
+      "clip_ratio/region_mean": 0.007816987024853006,
+      "completions/clipped_ratio": 0.5625,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3362.75,
+      "completions/min_length": 1841.5,
+      "epoch": 0.11258278145695365,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2688680589199066,
+      "kl": 0.19015406165271997,
+      "learning_rate": 4.65185506750986e-06,
+      "loss": -0.3238374888896942,
+      "reward": 0.37831513583660126,
+      "reward_std": 0.4376380145549774,
+      "rewards/MultiComponentReward/mean": 0.37831513583660126,
+      "rewards/MultiComponentReward/std": 0.4376379996538162,
+      "step": 51,
+      "step_time": 388.6353335408494
+    },
+    {
+      "clip_ratio/high_max": 0.0015778005181346089,
+      "clip_ratio/high_mean": 0.0015778005181346089,
+      "clip_ratio/low_mean": 0.0018030197243206203,
+      "clip_ratio/low_min": 0.0018030197243206203,
+      "clip_ratio/region_mean": 0.003380820242455229,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3641.5,
+      "completions/mean_length": 1865.5625,
+      "completions/min_length": 342.5,
+      "epoch": 0.11479028697571744,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.45149463415145874,
+      "kl": 0.2064807526767254,
+      "learning_rate": 4.638410650401267e-06,
+      "loss": -0.03052283078432083,
+      "reward": 0.8545674085617065,
+      "reward_std": 0.2037927806377411,
+      "rewards/MultiComponentReward/mean": 0.8545674085617065,
+      "rewards/MultiComponentReward/std": 0.2037927731871605,
+      "step": 52,
+      "step_time": 330.25200098752975
+    },
+    {
+      "clip_ratio/high_max": 0.004285238741431385,
+      "clip_ratio/high_mean": 0.004285238741431385,
+      "clip_ratio/low_mean": 0.0009839165140874684,
+      "clip_ratio/low_min": 0.0009839165140874684,
+      "clip_ratio/region_mean": 0.005269155255518854,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1958.5,
+      "completions/mean_length": 1569.1875,
+      "completions/min_length": 1136.5,
+      "epoch": 0.11699779249448124,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1865282654762268,
+      "kl": 0.10420421231538057,
+      "learning_rate": 4.62473173246716e-06,
+      "loss": -0.2261577546596527,
+      "reward": 0.9028474688529968,
+      "reward_std": 0.1216650977730751,
+      "rewards/MultiComponentReward/mean": 0.9028474688529968,
+      "rewards/MultiComponentReward/std": 0.121665108948946,
+      "step": 53,
+      "step_time": 191.50661983899772
+    },
+    {
+      "clip_ratio/high_max": 0.0016853307170094922,
+      "clip_ratio/high_mean": 0.0016853307170094922,
+      "clip_ratio/low_mean": 0.001708984375,
+      "clip_ratio/low_min": 0.001708984375,
+      "clip_ratio/region_mean": 0.0033943150920094922,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3044.0,
+      "completions/min_length": 770.0,
+      "epoch": 0.11920529801324503,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3003067672252655,
+      "kl": 0.06439034873619676,
+      "learning_rate": 4.610819813755038e-06,
+      "loss": 0.23977448046207428,
+      "reward": 0.5597200393676758,
+      "reward_std": 0.46622297167778015,
+      "rewards/MultiComponentReward/mean": 0.5597200393676758,
+      "rewards/MultiComponentReward/std": 0.46622298657894135,
+      "step": 54,
+      "step_time": 382.5643189586699
+    },
+    {
+      "clip_ratio/high_max": 0.005591174180153757,
+      "clip_ratio/high_mean": 0.005591174180153757,
+      "clip_ratio/low_mean": 0.0015268308343365788,
+      "clip_ratio/low_min": 0.0015268308343365788,
+      "clip_ratio/region_mean": 0.007118005014490336,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3091.0,
+      "completions/min_length": 859.5,
+      "epoch": 0.12141280353200883,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2259521186351776,
+      "kl": 0.12408451177179813,
+      "learning_rate": 4.596676419863561e-06,
+      "loss": -0.15562379360198975,
+      "reward": 0.43030790984630585,
+      "reward_std": 0.40197329223155975,
+      "rewards/MultiComponentReward/mean": 0.43030790984630585,
+      "rewards/MultiComponentReward/std": 0.40197327733039856,
+      "step": 55,
+      "step_time": 384.5411189426668
+    },
+    {
+      "clip_ratio/high_max": 0.0012758715311065316,
+      "clip_ratio/high_mean": 0.0012758715311065316,
+      "clip_ratio/low_mean": 0.0016176702920347452,
+      "clip_ratio/low_min": 0.0016176702920347452,
+      "clip_ratio/region_mean": 0.002893541823141277,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3289.5,
+      "completions/min_length": 2168.0,
+      "epoch": 0.12362030905077263,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17063800990581512,
+      "kl": 0.05120025109499693,
+      "learning_rate": 4.582303101775249e-06,
+      "loss": 0.33173251152038574,
+      "reward": 0.5529143065214157,
+      "reward_std": 0.4493340998888016,
+      "rewards/MultiComponentReward/mean": 0.5529143065214157,
+      "rewards/MultiComponentReward/std": 0.4493340998888016,
+      "step": 56,
+      "step_time": 390.2121843043715
+    },
+    {
+      "clip_ratio/high_max": 0.00523632307886146,
+      "clip_ratio/high_mean": 0.00523632307886146,
+      "clip_ratio/low_mean": 0.0006986957741901278,
+      "clip_ratio/low_min": 0.0006986957741901278,
+      "clip_ratio/region_mean": 0.005935018853051588,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2723.8125,
+      "completions/min_length": 533.0,
+      "epoch": 0.12582781456953643,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22646665573120117,
+      "kl": 0.2158638322725892,
+      "learning_rate": 4.567701435686405e-06,
+      "loss": -0.4513983428478241,
+      "reward": 0.6007745414972305,
+      "reward_std": 0.35604920983314514,
+      "rewards/MultiComponentReward/mean": 0.6007745414972305,
+      "rewards/MultiComponentReward/std": 0.35604920983314514,
+      "step": 57,
+      "step_time": 371.80605033878237
+    },
+    {
+      "clip_ratio/high_max": 0.002434744776110165,
+      "clip_ratio/high_mean": 0.002434744776110165,
+      "clip_ratio/low_mean": 0.0013598802397609688,
+      "clip_ratio/low_min": 0.0013598802397609688,
+      "clip_ratio/region_mean": 0.0037946250158711337,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3277.0,
+      "completions/mean_length": 1777.6875,
+      "completions/min_length": 1136.0,
+      "epoch": 0.1280353200883002,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2902727425098419,
+      "kl": 0.12625387962907553,
+      "learning_rate": 4.55287302283426e-06,
+      "loss": 0.10692867636680603,
+      "reward": 0.8373697400093079,
+      "reward_std": 0.2198527753353119,
+      "rewards/MultiComponentReward/mean": 0.8373697400093079,
+      "rewards/MultiComponentReward/std": 0.2198527790606022,
+      "step": 58,
+      "step_time": 300.5143364351243
+    },
+    {
+      "clip_ratio/high_max": 0.0034968220279552042,
+      "clip_ratio/high_mean": 0.0034968220279552042,
+      "clip_ratio/low_mean": 0.0007754346297588199,
+      "clip_ratio/low_min": 0.0007754346297588199,
+      "clip_ratio/region_mean": 0.004272256657714024,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3109.0,
+      "completions/mean_length": 1728.25,
+      "completions/min_length": 1099.5,
+      "epoch": 0.13024282560706402,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.28320446610450745,
+      "kl": 0.11147503461688757,
+      "learning_rate": 4.537819489321385e-06,
+      "loss": 0.029252953827381134,
+      "reward": 0.9200782179832458,
+      "reward_std": 0.13229025900363922,
+      "rewards/MultiComponentReward/mean": 0.9200782179832458,
+      "rewards/MultiComponentReward/std": 0.13229026645421982,
+      "step": 59,
+      "step_time": 282.09600766561925
+    },
+    {
+      "clip_ratio/high_max": 0.0034783274168148637,
+      "clip_ratio/high_mean": 0.0034783274168148637,
+      "clip_ratio/low_mean": 0.001953125,
+      "clip_ratio/low_min": 0.001953125,
+      "clip_ratio/region_mean": 0.005431452416814864,
+      "completions/clipped_ratio": 0.4375,
+      "completions/max_length": 3948.0,
+      "completions/mean_length": 3077.875,
+      "completions/min_length": 2020.5,
+      "epoch": 0.13245033112582782,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16196228563785553,
+      "kl": 0.15367124415934086,
+      "learning_rate": 4.522542485937369e-06,
+      "loss": -0.3941088914871216,
+      "reward": 0.43090952187776566,
+      "reward_std": 0.17558224871754646,
+      "rewards/MultiComponentReward/mean": 0.43090952187776566,
+      "rewards/MultiComponentReward/std": 0.17558224871754646,
+      "step": 60,
+      "step_time": 367.55937029374763
+    },
+    {
+      "clip_ratio/high_max": 0.004485186917008832,
+      "clip_ratio/high_mean": 0.004485186917008832,
+      "clip_ratio/low_mean": 0.001581493008416146,
+      "clip_ratio/low_min": 0.001581493008416146,
+      "clip_ratio/region_mean": 0.006066679925424978,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3242.5,
+      "completions/mean_length": 2011.5625,
+      "completions/min_length": 1232.0,
+      "epoch": 0.1346578366445916,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19294370710849762,
+      "kl": 0.1295578507706523,
+      "learning_rate": 4.507043687977787e-06,
+      "loss": -0.22440877556800842,
+      "reward": 0.930275559425354,
+      "reward_std": 0.06418891251087189,
+      "rewards/MultiComponentReward/mean": 0.930275559425354,
+      "rewards/MultiComponentReward/std": 0.06418891251087189,
+      "step": 61,
+      "step_time": 297.4619655525312
+    },
+    {
+      "clip_ratio/high_max": 0.009054507740074769,
+      "clip_ratio/high_mean": 0.009054507740074769,
+      "clip_ratio/low_mean": 0.000762939453125,
+      "clip_ratio/low_min": 0.000762939453125,
+      "clip_ratio/region_mean": 0.009817447193199769,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2536.5625,
+      "completions/min_length": 333.5,
+      "epoch": 0.1368653421633554,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22202427685260773,
+      "kl": 0.24569420609623194,
+      "learning_rate": 4.491324795060491e-06,
+      "loss": -0.27563101053237915,
+      "reward": 0.5970795005559921,
+      "reward_std": 0.3636050373315811,
+      "rewards/MultiComponentReward/mean": 0.5970795005559921,
+      "rewards/MultiComponentReward/std": 0.3636050671339035,
+      "step": 62,
+      "step_time": 378.6595879411325
+    },
+    {
+      "clip_ratio/high_max": 0.002085420419462025,
+      "clip_ratio/high_mean": 0.002085420419462025,
+      "clip_ratio/low_mean": 0.0012938255676999688,
+      "clip_ratio/low_min": 0.0012938255676999688,
+      "clip_ratio/region_mean": 0.003379245987161994,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 2527.0,
+      "completions/min_length": 959.5,
+      "epoch": 0.1390728476821192,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08440224826335907,
+      "kl": 0.2217963901348412,
+      "learning_rate": 4.475387530939226e-06,
+      "loss": -0.24322086572647095,
+      "reward": 0.7439667880535126,
+      "reward_std": 0.3273301273584366,
+      "rewards/MultiComponentReward/mean": 0.7439667880535126,
+      "rewards/MultiComponentReward/std": 0.3273301422595978,
+      "step": 63,
+      "step_time": 374.1972050392069
+    },
+    {
+      "clip_ratio/high_max": 0.004135866474825889,
+      "clip_ratio/high_mean": 0.004135866474825889,
+      "clip_ratio/low_mean": 0.0009211710712406784,
+      "clip_ratio/low_min": 0.0009211710712406784,
+      "clip_ratio/region_mean": 0.005057037546066567,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2753.0,
+      "completions/mean_length": 2028.8125,
+      "completions/min_length": 1423.0,
+      "epoch": 0.141280353200883,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2267836481332779,
+      "kl": 0.10806227196007967,
+      "learning_rate": 4.4592336433146e-06,
+      "loss": -0.40853649377822876,
+      "reward": 0.8472882509231567,
+      "reward_std": 0.059549352154135704,
+      "rewards/MultiComponentReward/mean": 0.8472882509231567,
+      "rewards/MultiComponentReward/std": 0.059549355879426,
+      "step": 64,
+      "step_time": 263.99467156175524
+    },
+    {
+      "clip_ratio/high_max": 0.003504310443531722,
+      "clip_ratio/high_mean": 0.003504310443531722,
+      "clip_ratio/low_mean": 0.002197265625,
+      "clip_ratio/low_min": 0.002197265625,
+      "clip_ratio/region_mean": 0.005701576068531722,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3217.0,
+      "completions/min_length": 798.0,
+      "epoch": 0.1434878587196468,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3381159007549286,
+      "kl": 0.16510179545730352,
+      "learning_rate": 4.442864903642428e-06,
+      "loss": 0.020503338426351547,
+      "reward": 0.4137403219938278,
+      "reward_std": 0.3740786761045456,
+      "rewards/MultiComponentReward/mean": 0.4137403219938278,
+      "rewards/MultiComponentReward/std": 0.3740786612033844,
+      "step": 65,
+      "step_time": 394.03293395834044
+    },
+    {
+      "clip_ratio/high_max": 0.004450821550562978,
+      "clip_ratio/high_mean": 0.004450821550562978,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004450821550562978,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1414.5,
+      "completions/mean_length": 1054.625,
+      "completions/min_length": 701.5,
+      "epoch": 0.1456953642384106,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.12737882137298584,
+      "kl": 0.22255737707018852,
+      "learning_rate": 4.426283106939474e-06,
+      "loss": -0.16745375096797943,
+      "reward": 0.9651360213756561,
+      "reward_std": 0.015341410413384438,
+      "rewards/MultiComponentReward/mean": 0.9651360213756561,
+      "rewards/MultiComponentReward/std": 0.015341411344707012,
+      "step": 66,
+      "step_time": 144.73526109801605
+    },
+    {
+      "clip_ratio/high_max": 0.003201823856215924,
+      "clip_ratio/high_mean": 0.003201823856215924,
+      "clip_ratio/low_mean": 0.0023111290647648275,
+      "clip_ratio/low_min": 0.0023111290647648275,
+      "clip_ratio/region_mean": 0.0055129529209807515,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 3155.0,
+      "completions/mean_length": 1831.0625,
+      "completions/min_length": 727.5,
+      "epoch": 0.1479028697571744,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5612183213233948,
+      "kl": 0.27365000173449516,
+      "learning_rate": 4.409490071586606e-06,
+      "loss": -0.005772307515144348,
+      "reward": 0.782143771648407,
+      "reward_std": 0.26549532637000084,
+      "rewards/MultiComponentReward/mean": 0.782143771648407,
+      "rewards/MultiComponentReward/std": 0.26549534499645233,
+      "step": 67,
+      "step_time": 295.5472778393887
+    },
+    {
+      "clip_ratio/high_max": 0.006920789659488946,
+      "clip_ratio/high_mean": 0.006920789659488946,
+      "clip_ratio/low_mean": 0.0004744651378132403,
+      "clip_ratio/low_min": 0.0004744651378132403,
+      "clip_ratio/region_mean": 0.0073952547973021865,
+      "completions/clipped_ratio": 0.3125,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 3174.6875,
+      "completions/min_length": 1843.5,
+      "epoch": 0.15011037527593818,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13284337520599365,
+      "kl": 0.13025074917823076,
+      "learning_rate": 4.3924876391293915e-06,
+      "loss": -0.45167821645736694,
+      "reward": 0.6101965606212616,
+      "reward_std": 0.39190903306007385,
+      "rewards/MultiComponentReward/mean": 0.6101965606212616,
+      "rewards/MultiComponentReward/std": 0.39190904796123505,
+      "step": 68,
+      "step_time": 381.54576131887734
+    },
+    {
+      "clip_ratio/high_max": 0.0018094074330292642,
+      "clip_ratio/high_mean": 0.0018094074330292642,
+      "clip_ratio/low_mean": 0.0057537677930668,
+      "clip_ratio/low_min": 0.0057537677930668,
+      "clip_ratio/region_mean": 0.007563175226096064,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 3595.0,
+      "completions/mean_length": 2244.5625,
+      "completions/min_length": 1391.5,
+      "epoch": 0.152317880794702,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4544171690940857,
+      "kl": 0.1460673250257969,
+      "learning_rate": 4.3752776740761495e-06,
+      "loss": 0.5760840177536011,
+      "reward": 0.8151208460330963,
+      "reward_std": 0.2074478343129158,
+      "rewards/MultiComponentReward/mean": 0.8151208460330963,
+      "rewards/MultiComponentReward/std": 0.2074478194117546,
+      "step": 69,
+      "step_time": 339.99933627434075
+    },
+    {
+      "clip_ratio/high_max": 0.0017436161288060248,
+      "clip_ratio/high_mean": 0.0017436161288060248,
+      "clip_ratio/low_mean": 0.00054931640625,
+      "clip_ratio/low_min": 0.00054931640625,
+      "clip_ratio/region_mean": 0.002292932535056025,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 4096.0,
+      "completions/mean_length": 1669.125,
+      "completions/min_length": 485.5,
+      "epoch": 0.1545253863134658,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2377484291791916,
+      "kl": 0.45143136754631996,
+      "learning_rate": 4.357862063693486e-06,
+      "loss": -0.24177709221839905,
+      "reward": 0.7517738044261932,
+      "reward_std": 0.39623187482357025,
+      "rewards/MultiComponentReward/mean": 0.7517738044261932,
+      "rewards/MultiComponentReward/std": 0.39623187482357025,
+      "step": 70,
+      "step_time": 370.3126081833616
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 300,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 5,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}